Merge branch 'main' into log-repository-everywhere

NEWS

@@ -1,5 +1,8 @@
 1.9.9.dev0
  * #635: Log the repository path or label on every relevant log message, not just some logs.
+ * #981: Fix "spot" check file count delta error.
+ * #982: Fix for borgmatic "exclude_patterns" and "exclude_from" recursing into excluded
+   subdirectories.
 
 1.9.8
  * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg.

borgmatic/actions/check.py

@@ -442,7 +442,7 @@ def collect_spot_check_archive_paths(
             config,
             local_borg_version,
             global_arguments,
-            path_format='{type} {path}{NL}',  # noqa: FS003
+            path_format='{type} {path}{NUL}',  # noqa: FS003
             local_path=local_path,
             remote_path=remote_path,
         )
@@ -538,7 +538,7 @@ def compare_spot_check_hashes(
                     local_borg_version,
                     global_arguments,
                     list_paths=source_sample_paths_subset,
-                    path_format='{xxh64} {path}{NL}',  # noqa: FS003
+                    path_format='{xxh64} {path}{NUL}',  # noqa: FS003
                     local_path=local_path,
                     remote_path=remote_path,
                 )

borgmatic/actions/create.py

@@ -62,7 +62,7 @@ def collect_patterns(config):
             )
             + tuple(
                 parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for exclude_line in config.get('exclude_patterns', ())
@@ -76,7 +76,7 @@ def collect_patterns(config):
             )
             + tuple(
                 parse_pattern(
-                    f'{borgmatic.borg.pattern.Pattern_type.EXCLUDE.value} {exclude_line.strip()}',
+                    f'{borgmatic.borg.pattern.Pattern_type.NO_RECURSE.value} {exclude_line.strip()}',
                     borgmatic.borg.pattern.Pattern_style.FNMATCH,
                 )
                 for filename in config.get('exclude_from', ())

borgmatic/borg/create.py

@@ -34,14 +34,16 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
 
     if patterns_file is None:
         patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
+        operation_name = 'Writing'
     else:
         patterns_file.write('\n')
+        operation_name = 'Appending'
 
     patterns_output = '\n'.join(
         f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}'
         for pattern in patterns
     )
-    logger.debug(f'Writing patterns to {patterns_file.name}:\n{patterns_output}')
+    logger.debug(f'{operation_name} patterns to {patterns_file.name}:\n{patterns_output}')
 
     patterns_file.write(patterns_output)
     patterns_file.flush()
@@ -324,7 +326,7 @@ def make_base_create_command(
                 tuple(
                     borgmatic.borg.pattern.Pattern(
                         special_file_path,
-                        borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+                        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
                         borgmatic.borg.pattern.Pattern_style.FNMATCH,
                     )
                     for special_file_path in special_file_paths

borgmatic/borg/list.py

@@ -131,7 +131,7 @@ def capture_archive_listing(
             borg_local_path=local_path,
             borg_exit_codes=config.get('borg_exit_codes'),
         )
-        .strip('\n')
+        .strip('\0')
         .split('\0')
     )
 

borgmatic/hooks/data_source/btrfs.py

@@ -151,7 +151,7 @@ def make_snapshot_exclude_pattern(subvolume_path):  # pragma: no cover
             subvolume_path.lstrip(os.path.sep),
             snapshot_directory,
         ),
-        borgmatic.borg.pattern.Pattern_type.EXCLUDE,
+        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
         borgmatic.borg.pattern.Pattern_style.FNMATCH,
     )
 

tests/unit/actions/test_create.py

@@ -48,9 +48,9 @@ def test_collect_patterns_parses_config_patterns():
 
 def test_collect_patterns_converts_exclude_patterns():
     assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == (
-        Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
-        Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
-        Pattern('**/baz', Pattern_type.EXCLUDE, Pattern_style.SHELL),
+        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL),
     )
 
 
@@ -90,22 +90,22 @@ def test_collect_patterns_reads_config_exclude_from_file():
         io.StringIO('/bar\n# comment\n\n   \n/baz')
     )
     flexmock(module).should_receive('parse_pattern').with_args(
-        '- /foo', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+        '! /foo', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
     flexmock(module).should_receive('parse_pattern').with_args(
-        '- /bar', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+        '! /bar', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
     flexmock(module).should_receive('parse_pattern').with_args('# comment').never()
     flexmock(module).should_receive('parse_pattern').with_args('').never()
     flexmock(module).should_receive('parse_pattern').with_args('   ').never()
     flexmock(module).should_receive('parse_pattern').with_args(
-        '- /baz', default_style=Pattern_style.FNMATCH
-    ).and_return(Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH))
+        '! /baz', default_style=Pattern_style.FNMATCH
+    ).and_return(Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH))
 
     assert module.collect_patterns({'exclude_from': ['file1.txt', 'file2.txt']}) == (
-        Pattern('/foo', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
-        Pattern('/bar', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
-        Pattern('/baz', Pattern_type.EXCLUDE, Pattern_style.FNMATCH),
+        Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
+        Pattern('/baz', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH),
     )
 
 

tests/unit/borg/test_create.py

@@ -661,7 +661,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
         (
             Pattern(
                 '/dev/null',
-                Pattern_type.EXCLUDE,
+                Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
             ),
         ),
@@ -711,7 +711,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
         (
             Pattern(
                 '/dev/null',
-                Pattern_type.EXCLUDE,
+                Pattern_type.NO_RECURSE,
                 Pattern_style.FNMATCH,
             ),
         ),

tests/unit/hooks/data_source/test_btrfs.py

@@ -218,7 +218,7 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns():
     ).and_return(
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -227,7 +227,7 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns():
     ).and_return(
         Pattern(
             '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -255,13 +255,13 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns():
         Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
         Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'),
         Pattern(
             '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
     ]
@@ -287,7 +287,7 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands():
     ).and_return(
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -312,7 +312,7 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands():
         Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
     ]
@@ -342,7 +342,7 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands():
     ).and_return(
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -367,7 +367,7 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands():
         Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
     ]
@@ -456,7 +456,7 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
     ).and_return(
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -465,7 +465,7 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
     ).and_return(
         Pattern(
             '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         )
     )
@@ -493,13 +493,13 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns():
         Pattern('/mnt/subvol1/.borgmatic-1234/mnt/subvol1'),
         Pattern(
             '/mnt/subvol1/.borgmatic-1234/mnt/subvol1/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
         Pattern('/mnt/subvol2/.borgmatic-1234/mnt/subvol2'),
         Pattern(
             '/mnt/subvol2/.borgmatic-1234/mnt/subvol2/.borgmatic-1234',
-            Pattern_type.EXCLUDE,
+            Pattern_type.NO_RECURSE,
             Pattern_style.FNMATCH,
         ),
     ]

tests/unit/hooks/monitoring/test_sentry.py

@@ -28,9 +28,9 @@ def test_ping_monitor_constructs_cron_url_and_pings_it(state, configured_states,
     if configured_states:
         hook_config['states'] = configured_states
 
-    flexmock(module.requests).should_receive('get').with_args(
+    flexmock(module.requests).should_receive('post').with_args(
         f'https://o294220.ingest.us.sentry.io/api/203069/cron/test/5f80ec/?status={expected_status}'
-    ).and_return(flexmock(ok=True))
+    ).and_return(flexmock(ok=True)).once()
 
     module.ping_monitor(
         hook_config,
@@ -48,7 +48,7 @@ def test_ping_monitor_with_unconfigured_state_bails():
         'monitor_slug': 'test',
         'states': ['fail'],
     }
-    flexmock(module.requests).should_receive('get').never()
+    flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
         hook_config,
@@ -67,7 +67,6 @@ def test_ping_monitor_with_unconfigured_state_bails():
         'https://o294220.ingest.us.sentry.io/203069',
         'https://5f80ec@/203069',
         'https://5f80ec@o294220.ingest.us.sentry.io',
-        'https://5f80ec@o294220.ingest.us.sentry.io/203069/',
     ),
 )
 def test_ping_monitor_with_invalid_data_source_name_url_bails(data_source_name_url):
@@ -76,7 +75,7 @@ def test_ping_monitor_with_invalid_data_source_name_url_bails(data_source_name_u
         'monitor_slug': 'test',
     }
 
-    flexmock(module.requests).should_receive('get').never()
+    flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
         hook_config,
@@ -95,7 +94,7 @@ def test_ping_monitor_with_invalid_sentry_state_bails():
         # This should never actually happen in practice, because the config schema prevents it.
         'states': ['log'],
     }
-    flexmock(module.requests).should_receive('get').never()
+    flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
         hook_config,
@@ -112,7 +111,7 @@ def test_ping_monitor_with_dry_run_bails():
         'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
         'monitor_slug': 'test',
     }
-    flexmock(module.requests).should_receive('get').never()
+    flexmock(module.requests).should_receive('post').never()
 
     module.ping_monitor(
         hook_config,
@@ -122,3 +121,27 @@ def test_ping_monitor_with_dry_run_bails():
         monitoring_log_level=1,
         dry_run=True,
     )
+
+
+def test_ping_monitor_with_network_error_does_not_raise():
+    hook_config = {
+        'data_source_name_url': 'https://5f80ec@o294220.ingest.us.sentry.io/203069',
+        'monitor_slug': 'test',
+    }
+
+    response = flexmock(ok=False)
+    response.should_receive('raise_for_status').and_raise(
+        module.requests.exceptions.ConnectionError
+    )
+    flexmock(module.requests).should_receive('post').with_args(
+        'https://o294220.ingest.us.sentry.io/api/203069/cron/test/5f80ec/?status=in_progress'
+    ).and_return(response).once()
+
+    module.ping_monitor(
+        hook_config,
+        {},
+        'config.yaml',
+        borgmatic.hooks.monitoring.monitor.State.START,
+        monitoring_log_level=1,
+        dry_run=False,
+    )