Fix traceback with "create" action and "--json" flag when a database hook is configured (#563).

NEWS

@@ -3,6 +3,7 @@
  * #560: Fix all database hooks to error when the requested database to restore isn't present in the
    Borg archive.
  * #561: Fix command-line "--override" flag to continue supporting old configuration file formats.
+ * #563: Fix traceback with "create" action and "--json" flag when a database hook is configured.
 
 1.6.5
  * #553: Fix logging to include the full traceback when Borg experiences an internal error, not just

borgmatic/execute.py

@@ -51,6 +51,9 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     process with the requested log level. Additionally, raise a CalledProcessError if a process
     exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
 
+    If output log level is None, then instead of logging, capture output for each process and return
+    it as a dict from the process to its output.
+
     For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
     stdouts are given to exclude, then for any matching processes, log from their stderr instead.
 
@@ -65,6 +68,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if process.stdout or process.stderr
     }
     output_buffers = list(process_for_output_buffer.keys())
+    captured_outputs = collections.defaultdict(list)
 
     # Log output for each process until they all exit.
     while True:
@@ -99,7 +103,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
                     if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
                         last_lines.pop(0)
 
-                    logger.log(output_log_level, line)
+                    if output_log_level is None:
+                        captured_outputs[ready_process].append(line)
+                    else:
+                        logger.log(output_log_level, line)
 
         still_running = False
 
@@ -133,6 +140,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if not still_running:
             break
 
+    if captured_outputs:
+        return {
+            process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
+        }
+
 
 def log_command(full_command, input_file, output_file):
     '''
@@ -222,13 +234,14 @@ def execute_command_with_processes(
     run as well. This is useful, for instance, for processes that are streaming output to a named
     pipe that the given command is consuming from.
 
-    If an open output file object is given, then write stdout to the file and only log stderr (but
-    only if an output log level is set). If an open input file object is given, then read stdin from
-    the file.  If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If a Borg local path is given, then for any matching command or process (regardless of
-    arguments), treat exit code 1 as a warning instead of an error.
+    If an open output file object is given, then write stdout to the file and only log stderr. But
+    if output log level is None, instead suppress logging and return the captured output for (only)
+    the given command. If an open input file object is given, then read stdin from the file. If
+    shell is True, execute the command within a shell. If an extra environment dict is given, then
+    use it to augment the current environment, and pass the result into the command. If a working
+    directory is given, use that as the present working directory when running the command. If a
+    Borg local path is given, then for any matching command or process (regardless of arguments),
+    treat exit code 1 as a warning instead of an error.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
@@ -259,9 +272,12 @@ def execute_command_with_processes(
                 process.kill()
         raise
 
-    log_outputs(
+    captured_outputs = log_outputs(
         tuple(processes) + (command_process,),
         (input_file, output_file),
         output_log_level,
         borg_local_path=borg_local_path,
     )
+
+    if output_log_level is None:
+        return captured_outputs.get(command_process)

tests/integration/test_execute.py

@@ -54,6 +54,30 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
     )
 
 
+def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
+    flexmock(module.logger).should_receive('log').never()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
+
+    there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    captured_outputs = module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
+        output_log_level=None,
+        borg_local_path='borg',
+    )
+
+    assert captured_outputs == {hi_process: 'hi', there_process: 'there'}
+
+
 def test_log_outputs_includes_error_output_in_exception():
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)

tests/unit/test_execute.py

@@ -289,6 +289,27 @@ def test_execute_command_with_processes_calls_full_command():
     assert output is None
 
 
+def test_execute_command_with_processes_returns_output_with_output_log_level_none():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    process = flexmock(stdout=None)
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(process).once()
+    flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
+
+    output = module.execute_command_with_processes(full_command, processes, output_log_level=None)
+
+    assert output == 'out'
+
+
 def test_execute_command_with_processes_calls_full_command_with_output_file():
     full_command = ['foo', 'bar']
     processes = (flexmock(),)