diff --git a/.travis.yml b/.travis.yml index ea553d96c..d18eef66e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,7 @@ python: - "3.6" - "3.9" - "3.10" - - "3.9-dev" - - "nightly" + - "3.11" before_install: # disable mongodb as we don't need it and it sometimes temporary fails @@ -32,16 +31,15 @@ before_install: jobs: exclude: - arch: ppc64le - python: "3.9" + python: "3.6" - arch: ppc64le - python: "3.9-dev" + python: "3.10" - arch: ppc64le - python: "nightly" - # Disable Python "3.9-dev" and "nightly" testing for AMD64, too. + python: "3.11" - arch: amd64 - python: "3.9-dev" + python: "3.9" - arch: amd64 - python: "nightly" + python: "3.11" install: - pip install coveralls pyfakefs diff --git a/common/config.py b/common/config.py index 675566784..9dc563f07 100644 --- a/common/config.py +++ b/common/config.py @@ -1,5 +1,6 @@ # Back In Time -# Copyright (C) 2008-2022 Oprea Dan, Bart de Koning, Richard Bailey, Germar Reitze +# Copyright (C) 2008-2022 Oprea Dan, Bart de Koning, Richard Bailey, +# Germar Reitze # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,6 +16,18 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""Configuration logic. + +This module and its `Config` class contain the appliation logic handling the +configuration of Back In Time. The handling of the configuration file itself +is separated in the module :py:module:`configfile`. + +Development notes: + Some of the methods have code comments starting with `#? ` instead of + `# `. These special comments are used to generate the manpage + `backintime-config`. The script `create-manpage-backintime-config.py` + parses this module for that. +""" import os import sys @@ -445,13 +458,8 @@ def setSnapshotsPath(self, value, profile_id = None, mode = None): return True def snapshotsMode(self, profile_id=None): - """Use mode (or backend) for this snapshot. - - Look at 'man backintime' section 'Modes'. - - Returns: - str: Possible values are local|local_encfs|ssh|ssh_encfs. - """ + #? Use mode (or backend) for this snapshot. Look at 'man backintime' + #? section 'Modes'.;local|local_encfs|ssh|ssh_encfs return self.profileStrValue('snapshots.mode', 'local', profile_id) def setSnapshotsMode(self, value, profile_id = None): @@ -850,8 +858,12 @@ def setScheduleMode(self, value, profile_id = None): self.setProfileIntValue('schedule.mode', value, profile_id) def scheduleTime(self, profile_id = None): - #?What time the cronjob should run? Only valid for - #?\fIprofile.schedule.mode\fR >= 20;0-24 + #?Position-coded number with the format "hhmm" to specify the hour + #?and minute the cronjob should start (eg. 2015 means a quarter + #?past 8pm). Leading zeros can be omitted (eg. 30 = 0030). + #?Only valid for + #?\fIprofile.schedule.mode\fR = 20 (daily), 30 (weekly), + #?40 (monthly) and 80 (yearly);0-2400 return self.profileIntValue('schedule.time', 0, profile_id) def setScheduleTime(self, value, profile_id = None): diff --git a/common/create-manpage-backintime-config.py b/common/create-manpage-backintime-config.py index eb0204b50..519327b8f 100644 --- a/common/create-manpage-backintime-config.py +++ b/common/create-manpage-backintime-config.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Back In Time # Copyright (C) 2012-2022 Germar Reitze # @@ -15,6 +16,37 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""This script is a helper to create a manpage about Back In Times's config +file. + +The file `common/config.py` is parsed for variable names, default values and +other informations. The founder of that script @Germar gave a detailed +description about that script in #1354. + +The script reads every line and tries to analyse it: + - It searches for `DEFAULT` and puts those into a `dict` for later replacing + the variable with the value. + - If that didn't match it will look for lines starting with `#?` which is + basically my own description for the manpage-entry. + Multiple lines will get merged and stored in `commentline` until the + processing of the current config option is done. That will reset + `commentline`. + - If a line starts with `#` it will be skipped. + - Next the script searches for lines which ``return`` the config value (like + `snapshots.ssh.port`. There it will extract the + key/name (`snapshots.ssh.port`), the default value (`22`), + the instance (`Int`) and if it is a profile or general value. + - If the line contains a `List` value like `snapshots.include` it will + process all values for the list like `snapshots.include..value` and + `snapshots.include..type`. Also it will add the size like + `snapshots.include.size`. + +In `process_line` it will replace some information with those I wrote manually +in the `#?` description, seperated by `;` there is the comment, value, +force_default and force_var. If there is no forced value it will chose the +value based on the instance with `select_values` +""" + import re import os import sys @@ -23,14 +55,15 @@ PATH = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])) CONFIG = os.path.join(PATH, 'config.py') -MAN = os.path.join(PATH, 'man/C/backintime-config.1') +MAN = os.path.join(PATH, 'man/C/backintime-config.1') + with open(os.path.join(PATH, '../VERSION'), 'r') as f: VERSION = f.read().strip('\n') -SORT = True #True = sort by alphabet; False = sort by line numbering + +SORT = True # True = sort by alphabet; False = sort by line numbering c_list = re.compile(r'.*?self\.(?!set)((?:profile)?)(List)Value ?\( ?[\'"](.*?)[\'"], ?((?:\(.*\)|[^,]*)), ?[\'"]?([^\'",\)]*)[\'"]?') -c = re.compile(r'.*?self\.(?!set)((?:profile)?)(.*?)Value ?\( ?[\'"](.*?)[\'"] ?(%?[^,]*?), ?[\'"]?([^\'",\)]*)[\'"]?') -c_default = re.compile(r'(^DEFAULT[\w]*|CONFIG_VERSION)[\s]*= (.*)') +c = re.compile(r'.*?self\.(?!set)((?:profile)?)(.*?)Value ?\( ?[\'"](.*?)[\'"] ?(%?[^,]*?), ?[\'"]?([^\'",\)]*)[\'"]?') HEADER = '''.TH backintime-config 1 "%s" "version %s" "USER COMMANDS" .SH NAME @@ -63,150 +96,267 @@ This manual page was written by BIT Team(). ''' -INSTANCE = 'instance' -NAME = 'name' -VALUES = 'values' -DEFAULT = 'default' -COMMENT = 'comment' +INSTANCE = 'instance' +NAME = 'name' +VALUES = 'values' +DEFAULT = 'default' +COMMENT = 'comment' REFERENCE = 'reference' -LINE = 'line' +LINE = 'line' + -def output(instance = '', name = '', values = '', default = '', comment = '', reference = '', line = 0): +def output(instance='', name='', values='', default='', + comment='', reference='', line=0): + """ + """ if not default: default = "''" - ret = '.IP "\\fI%s\\fR" 6\n' % name + + ret = '.IP "\\fI%s\\fR" 6\n' % name ret += '.RS\n' - ret += 'Type: %-10sAllowed Values: %s\n' %(instance.lower(), values) + ret += 'Type: %-10sAllowed Values: %s\n' % (instance.lower(), values) ret += '.br\n' ret += '%s\n' % comment ret += '.PP\n' + if SORT: ret += 'Default: %s\n' % default else: ret += 'Default: %-18s %s line: %d\n' % (default, reference, line) + ret += '.RE\n' + return ret + def select(a, b): if a: return a + return b + def select_values(instance, values): if values: return values - if instance.lower() == 'bool': - return 'true|false' - if instance.lower() == 'str': - return 'text' - if instance.lower() == 'int': - return '0-99999' - -def process_line(d, key, profile, instance, name, var, default, commentline, values, force_var, force_default, replace_default, counter): - #Ignore commentlines with #?! and 'config.version' + + return { + 'bool': 'true|false', + 'str': 'text', + 'int': '0-99999' + }[instance.lower()] + + +def process_line(d, key, profile, instance, name, var, default, commentline, + values, force_var, force_default, replace_default, counter): + # Ignore commentlines with #?! and 'config.version' comment = None - if not commentline.startswith('!') and not key in d: + + if not commentline.startswith('!') and key not in d: d[key] = {} commentline = commentline.split(';') + try: - comment = commentline[0] - values = commentline[1] + comment = commentline[0] + values = commentline[1] force_default = commentline[2] - force_var = commentline[3] + force_var = commentline[3] + except IndexError: pass if default.startswith('self.') and default[5:] in replace_default: default = replace_default[default[5:]] - if isinstance(force_default, str) and force_default.startswith('self.') and force_default[5:] in replace_default: + if isinstance(force_default, str) \ + and force_default.startswith('self.') \ + and force_default[5:] in replace_default: force_default = replace_default[force_default[5:]] if instance.lower() == 'bool': default = default.lower() - d[key][INSTANCE] = instance - d[key][NAME] = re.sub(r'%[\S]', '<%s>' % select(force_var, var).upper(), name) - d[key][VALUES] = select_values(instance, values) - d[key][DEFAULT] = select(force_default, default) - d[key][COMMENT] = re.sub(r'\\n', '\n.br\n', comment) + + d[key][INSTANCE] = instance + d[key][NAME] = re.sub( + r'%[\S]', '<%s>' % select(force_var, var).upper(), name + ) + d[key][VALUES] = select_values(instance, values) + d[key][DEFAULT] = select(force_default, default) + d[key][COMMENT] = re.sub(r'\\n', '\n.br\n', comment) d[key][REFERENCE] = 'config.py' - d[key][LINE] = counter + d[key][LINE] = counter + def main(): + d = { + 'profiles.version': { + INSTANCE: 'int', + NAME: 'profiles.version', + VALUES: '1', + DEFAULT: '1', + COMMENT: 'Internal version of profiles config.', + REFERENCE: 'configfile.py', + LINE: 419 + }, + 'profiles': { + INSTANCE: 'str', + NAME: 'profiles', + VALUES: 'int separated by colon (e.g. 1:3:4)', + DEFAULT: '1', + COMMENT: 'All active Profiles ( in profile.snapshots...).', + REFERENCE: 'configfile.py', + LINE: 472 + }, + 'profile.name': { + INSTANCE: 'str', + NAME: 'profile.name', + VALUES: 'text', + DEFAULT: 'Main profile', + COMMENT: 'Name of this profile.', + REFERENCE: 'configfile.py', + LINE: 704 + } + } + + # Default variables and there values collected from config.py replace_default = {} - d = {} - d['profiles.version'] = {INSTANCE : 'int', - NAME : 'profiles.version', - VALUES : '1', - DEFAULT : '1', - COMMENT : 'Internal version of profiles config.', - REFERENCE : 'configfile.py', - LINE : 419} - d['profiles'] = {INSTANCE : 'str', - NAME : 'profiles', - VALUES : 'int separated by colon (e.g. 1:3:4)', - DEFAULT : '1', - COMMENT : 'All active Profiles ( in profile.snapshots...).', - REFERENCE : 'configfile.py', - LINE : 472} - d['profile.name'] = {INSTANCE : 'str', - NAME : 'profile.name', - VALUES : 'text', - DEFAULT : 'Main profile', - COMMENT : 'Name of this profile.', - REFERENCE : 'configfile.py', - LINE : 704} + + # Variables named "CONFIG_VERSION" or with names starting with "DEFAULT" + regex_default = re.compile(r'(^DEFAULT[\w]*|CONFIG_VERSION)[\s]*= (.*)') + with open(CONFIG, 'r') as f: + print(f'Read "{CONFIG}".') commentline = '' - values = force_var = force_default = instance = name = var = default = None + values = force_var = force_default = instance \ + = name = var = default = None + for counter, line in enumerate(f, 1): line = line.lstrip() - m_default = c_default.match(line) + + # parse for DEFAULT variable + m_default = regex_default.match(line) + + # DEFAULT variable if m_default: - replace_default[m_default.group(1)] = m_default.group(2).replace('\\$', '\\\$') + replace_default[m_default.group(1)] \ + = m_default.group(2).replace('\\$', '\\\$') continue + + # Comment intended to use for the manpage if line.startswith('#?'): - if commentline and not ';' in commentline and not commentline.endswith('\\n'): + if commentline \ + and ';' not in commentline \ + and not commentline.endswith('\\n'): commentline += ' ' + commentline += line.lstrip('#?').rstrip('\n') + continue + + # Simple comments are ignored if line.startswith('#'): commentline = '' continue + # m = c_list_tuple.match(line) # if not m: + + # e.g. "return self.profileListValue('snapshots.include', ('str:value', 'int:type'), [], profile_id)" + # becomes + # ('profile', 'List', 'snapshots.include', "('str:value', 'int:type')", '[]') m = c_list.match(line) + if not m: + # e.g. "return self.profileBoolValue('snapshots.use_checksum', False, profile_id)" + # becomes + # ('profile', 'Bool', 'snapshots.use_checksum', '', 'False') m = c.match(line) + if m: profile, instance, name, var, default = m.groups() + if profile == 'profile': name = 'profile.' + name + var = var.lstrip('% ') + if instance.lower() == 'list': type_key = [x.strip('"\'') for x in re.findall(r'["\'].*?["\']', var)] commentline_split = commentline.split('::') + for i, tk in enumerate(type_key): - t, k = tk.split(':', maxsplit = 1) - process_line(d, key, profile, 'int', '%s.size' %name, var, '\-1', 'Quantity of %s. entries.' %name, values, force_var, force_default, replace_default, counter) - key = '%s.%s' %(name, k) + t, k = tk.split(':', maxsplit=1) + + process_line( + d, + key, + profile, + 'int', + '%s.size' % name, + var, + '\-1', + 'Quantity of %s. entries.' % name, + values, + force_var, + force_default, + replace_default, + counter) + + key = '%s.%s' % (name, k) key = key.lower() - process_line(d, key, profile, t, '%s..%s' %(name, k), var, '', commentline_split[i], values, force_var, force_default, replace_default, counter) + + process_line( + d, + key, + profile, + t, + '%s..%s' % (name, k), + var, + '', + commentline_split[i], + values, + force_var, + force_default, + replace_default, + counter + ) + else: key = re.sub(r'%[\S]', var, name).lower() - process_line(d, key, profile, instance, name, var, default, commentline, values, force_var, force_default, replace_default, counter) - values = force_var = force_default = instance = name = var = default = None + process_line( + d, + key, + profile, + instance, + name, + var, + default, + commentline, + values, + force_var, + force_default, + replace_default, + counter + ) + + values = force_var = force_default = instance \ + = name = var = default = None + commentline = '' with open(MAN, 'w') as f: + print(f'Write into "{MAN}".') f.write(HEADER) + if SORT: s = lambda x: x else: s = lambda x: d[x][LINE] - f.write('\n'.join(output(**d[key]) for key in sorted(d, key = s))) + + f.write('\n'.join(output(**d[key]) for key in sorted(d, key=s))) f.write(FOOTER) + if __name__ == '__main__': main() diff --git a/common/man/C/backintime-config.1 b/common/man/C/backintime-config.1 index a4c367384..974fe1aad 100644 --- a/common/man/C/backintime-config.1 +++ b/common/man/C/backintime-config.1 @@ -1,4 +1,4 @@ -.TH backintime-config 1 "Oct 2022" "version 1.3.3-dev" "USER COMMANDS" +.TH backintime-config 1 "Nov 2022" "version 1.3.3-dev" "USER COMMANDS" .SH NAME config \- BackInTime configuration files. .SH SYNOPSIS @@ -23,7 +23,7 @@ Run 'backintime check-config' to verify the configfile, create the snapshot fold .RS Type: int Allowed Values: 0-99999 .br -Internal version of current config + .PP Default: 6 .RE @@ -148,9 +148,7 @@ Default: 20 .RS Type: int Allowed Values: 0-2400 .br -Position-coded number with the format "hhmm" to specify the hour and minute the cronjob should start (eg. 2015 means a quarter past 8 pm). Leading zeros can be omitted (eg. 30 = 0030). -.br -Only valid for \fIprofile.schedule.mode\fR = 20 (daily), 30 (weekly), 40 (monthly) and 80 (yearly) +Position-coded number with the format "hhmm" to specify the hour and minute the cronjob should start (eg. 2015 means a quarter past 8pm). Leading zeros can be omitted (eg. 30 = 0030). Only valid for \fIprofile.schedule.mode\fR = 20 (daily), 30 (weekly), 40 (monthly) and 80 (yearly) .PP Default: 0 .RE @@ -419,7 +417,7 @@ Default: 1 .RS Type: str Allowed Values: local|local_encfs|ssh|ssh_encfs .br -Use mode (or backend) for this snapshot. Look at 'man backintime' section 'Modes'. + Use mode (or backend) for this snapshot. Look at 'man backintime' section 'Modes'. .PP Default: local .RE @@ -733,7 +731,7 @@ Type: str Allowed Values: text .br Prefix to run before every command on remote host. Variables need to be escaped with \\$FOO. This doesn't touch rsync. So to add a prefix for rsync use \fIprofile.snapshots.rsync_options.value\fR with --rsync-path="FOO=bar:\\$FOO /usr/bin/rsync" .PP -Default: 'PATH=/opt/bin:/opt/sbin:\\$PATH' +Default: 'PATH=/opt/bin:/opt/sbin:\\\$PATH' .RE .IP "\fIprofile.snapshots.ssh.private_key_file\fR" 6