rm custom logging

Author: pdimens

harpy/commands/view.py

@@ -12,8 +12,10 @@
 import rich_click as click
 from rich.console import Console
 from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
 from rich import print as rprint
-from harpy.common.printing import print_error
+from harpy.common.printing import harpy_table, print_error
 from harpy.common.file_ops import is_gzip
 
 def check_terminal_colors():
@@ -41,7 +43,7 @@ def check_terminal_colors():
 
 def parse_file(infile: str):
     '''
-    take a list of input file name, get the most recent by modificiation time, and print it via pygmentized less
+    take an input file name, get the most recent by modificiation time, and print it via pygmentized less
     returns a string of the file that was viewed
     '''
     if not os.access(infile, os.R_OK):
@@ -162,14 +164,15 @@ def environments(program):
     return
 
 @click.command(no_args_is_help = True, context_settings={"allow_interspersed_args" : False})
+@click.option("-c", "--choose", is_flag=True, default=False, help = "From one from all the logs")
 @click.argument('directory', required=True, type=click.Path(exists=True, file_okay=False), nargs=1)
-def log(directory):
+def log(directory, choose):
     """
     View a workflow's last log file
     
     The log file contains everything Snakemake printed during runtime.
     The only required input is the output folder previously created by Harpy where you can find
-    `logs/snakemake/`. Navigate with the typical `less` keyboard bindings, e.g.:
+    `.snakemake/log`. Navigate with the typical `less` keyboard bindings, e.g.:
     
     | key                     | function                   |
     | :---------------------- | :------------------------- |
@@ -178,7 +181,7 @@ def log(directory):
     | `/` + `pattern`         | search for `pattern`       |
     | `q`                     | exit                       |
     """
-    err_dir = os.path.join(directory, "logs", "snakemake")
+    err_dir = os.path.join(directory, ".snakemake", "log")
     err_file = "There are no log files"
     if not os.path.exists(err_dir):
         print_error(
@@ -191,7 +194,30 @@ def log(directory):
             "files not found", 
             f"{err_file} in [blue]{err_dir}[/]. Please check that this is the correct folder."
         )
-    target_file = sorted(files, key = os.path.getmtime)[-1]
+
+    files = sorted(files, key = os.path.getmtime, reverse = True)
+    if choose:
+        console = Console()
+        console.print()
+        console.rule('Snakemake Log Files', style = "green")
+        _tb = harpy_table()
+        _tb.add_column("Number", justify="left", style="bold green", no_wrap=True)
+        _tb.add_column("File", justify="left")
+        for i,j in enumerate(files,1):
+            _tb.add_row(str(i), os.path.basename(j))
+        console.print(_tb)
+        console.rule('[dim]sorted newest (top) to oldest (bottom)[/]', style = 'dim')
+        selection = Prompt.ask(
+            "\n[bold blue]Select a log file by number[/]",
+            choices=list(str(i) for i in range(1,len(files) + 1)),
+            show_choices=False
+        )
+        
+        selected_idx = int(selection) - 1
+        target_file = files[selected_idx]
+    else:
+        target_file = files[0]
+
     parse_file(target_file)
     rprint(
         Panel(

harpy/common/file_ops.py

@@ -22,6 +22,14 @@ def gzip_file(infile: str) -> None:
             shutil.copyfileobj(f_in, f_out)
         os.remove(infile)
 
+def last_sm_log(directory) -> str:
+    '''Find and return the name of the last snakemake log file, otherwise return an empty string'''
+    err_dir = os.path.join(directory, ".snakemake", "log")
+    if not os.path.exists(err_dir) or not os.path.exists(directory):
+        return ""
+    files = [i for i in glob.iglob(f"{err_dir}/*.log*")]        
+    return os.path.basename(sorted(files, key = os.path.getmtime)[-1])
+
 def purge_empty_logs(output_directory):
     """scan target_dir and remove empty files, then scan it again and remove empty directories"""
     for logfile in glob.glob(f"{output_directory}/logs/**/*", recursive = True):

harpy/common/launch.py

@@ -8,7 +8,7 @@
 from rich import box
 from rich.syntax import Syntax
 from rich.table import Table
-from harpy.common.file_ops import gzip_file, purge_empty_logs
+from harpy.common.file_ops import last_sm_log, purge_empty_logs
 from harpy.common.printing import CONSOLE, print_onerror, print_setup_error
 from harpy.common.progress import harpy_progressbar, harpy_pulsebar, harpy_progresspanel
 
@@ -92,7 +92,7 @@ def highlight_params(text: str):
         return f"\n[blue]{text}[/]"
     return text
 
-def launch_snakemake(sm_args, outdir, sm_logfile, quiet, CONSOLE = CONSOLE):
+def launch_snakemake(sm_args, outdir, quiet, CONSOLE = CONSOLE):
     """launch snakemake with the given commands"""
     exitcode = None
     sm_start = datetime.now()
@@ -227,7 +227,7 @@ def launch_snakemake(sm_args, outdir, sm_logfile, quiet, CONSOLE = CONSOLE):
             if exitcode in (1,2):
                 print_setup_error(exitcode)
             elif exitcode == 3:
-                print_onerror(os.path.join(os.path.basename(outdir), sm_logfile), datetime.now() - sm_start)
+                print_onerror(last_sm_log(outdir), datetime.now() - sm_start)
 
             CONSOLE.tab_size = 4
             CONSOLE._highlight = False
@@ -302,6 +302,5 @@ def launch_snakemake(sm_args, outdir, sm_logfile, quiet, CONSOLE = CONSOLE):
         CONSOLE.rule("[bold]Terminating Harpy", style = "yellow")
         process.terminate()
         process.wait()
-        gzip_file(os.path.join(outdir,sm_logfile))
         purge_empty_logs(outdir)
         sys.exit(1)

harpy/common/printing.py

@@ -12,6 +12,10 @@
 
 CONSOLE = Console(stderr=True, log_path=False)
 
+def harpy_table(title = None, caption = None):
+    '''Insantiate a generic but standardized table style for harpy output'''
+    return Table(title = title, caption = caption, show_header=False,pad_edge=False, show_edge=False, padding = (0,0), box=box.SIMPLE)
+
 def print_error(
     errortitle: str,
     errortext: RenderableType|str,
@@ -106,7 +110,7 @@ def print_onerror(logfile: str, time) -> None:
     minutes = (seconds % 3600) // 60
     seconds = seconds % 60
     time_text = f"{days} days, {hours} hours, {minutes} minutes, {seconds} seconds"
-    datatable = Table(show_header=False,pad_edge=False, show_edge=False, padding = (0,0), box=box.SIMPLE)
+    datatable = harpy_table()
     datatable.add_column("detail", justify="left", style="red", no_wrap=True)
     datatable.add_column("value", justify="left")
     datatable.add_row("Duration:", time_text)
@@ -117,13 +121,7 @@ def print_onerror(logfile: str, time) -> None:
     CONSOLE.rule("[bold]Where Error Occurred", style = "red")
 
 def print_shellcmd_simple(text):
-    _table = Table(
-        show_header=False,
-        pad_edge=False,
-        show_edge=False,
-        padding=(0,0),
-        box=box.SIMPLE,
-    )
+    _table = harpy_table()
     _table.add_column("Lpadding", justify="left")
     _table.add_column("shell", justify="left")
     _table.add_column("Rpadding", justify="left")
@@ -138,7 +136,7 @@ def workflow_info(*arg: tuple[str, str | int | float]|None) -> Table:
     Accepts an unlimited number of length-2 lists or tuples and returns a rich.Table with the value of the first indices as the row names and the second indices as the values
     Use None instead of a list to ignore that entry (useful for conditionals). The second value will always be converted to a string.
     """
-    table = Table(show_header=False,pad_edge=False, show_edge=False, padding = (0,0), box=box.SIMPLE)
+    table = harpy_table()
     table.add_column("detail", justify="left", style="light_steel_blue", no_wrap=True)
     table.add_column("value", justify="left")
     for i in arg:

harpy/common/workflow.py

@@ -1,7 +1,6 @@
 """Contains the WorkFlow class"""
 
 from datetime import datetime
-import glob
 import importlib.resources as resources
 import os
 import shutil
@@ -10,11 +9,10 @@
 import urllib.request
 import urllib.error
 import yaml
-from rich import box
 from rich.table import Table
 from harpy.common.conda import create_conda_recipes
-from harpy.common.file_ops import filepath, gzip_file, purge_empty_logs
-from harpy.common.printing import CONSOLE, print_error
+from harpy.common.file_ops import filepath, last_sm_log, purge_empty_logs
+from harpy.common.printing import CONSOLE, harpy_table, print_error
 from harpy.common.launch import launch_snakemake
 from harpy.common.summaries import Summary
 
@@ -30,7 +28,6 @@ def __init__(self, name, snakefile, outdir, container, quiet, inputdir = False):
         self.name: str = name
         self.output_directory: str = outdir
         self.workflow_directory = os.path.join(outdir, 'workflow')
-        self.snakemake_logfile: str = self.snakemake_log(outdir, name)
         self.snakemake_cmd_absolute: str = ""
         self.snakemake_cmd_relative: str = ""
         self.snakefile: str = snakefile
@@ -48,16 +45,6 @@ def __init__(self, name, snakefile, outdir, container, quiet, inputdir = False):
         self.summary: str = name.replace("_",".").replace(" ",".") + ".summary"
         self.summary_text: str = ""
 
-    def snakemake_log(self, outdir: str, workflow: str) -> str:
-        """Return a snakemake logfile name. Iterates logfile run number if one exists."""
-        attempts = glob.glob(os.path.join(outdir, "logs", "snakemake", "*.log*"))
-        timestamp = datetime.now().strftime("%d_%m_%Y") + ".log"
-        if not attempts:
-            os.makedirs(os.path.join(outdir, "logs", "snakemake"), exist_ok=True)
-            return os.path.join("logs", "snakemake", f"{workflow}.1.{timestamp}")
-        increment = sorted([int(i.split(".")[1]) for i in attempts])[-1] + 1
-        return os.path.join("logs", "snakemake", f"{workflow}.{increment}.{timestamp}")
-
     def setup_snakemake(self, threads: int, hpc: str|None = None, sm_extra: str|None = None):
         """
         Sets up the snakemake command based on hpc, threads, and extra snakemake params.
@@ -214,7 +201,6 @@ def write_workflow_config(self) -> None:
         are expected to be a dict
         """
         self.config["snakemake"] = {
-            "log" : self.snakemake_logfile,
             "absolute": self.snakemake_cmd_absolute,
             "relative": self.snakemake_cmd_relative,
             "conda_envs": self.conda
@@ -249,13 +235,15 @@ def print_onsuccess(self):
             return
         _relpath = os.path.relpath(self.output_directory)
         time_text = self.time_elapsed()
-        datatable = Table(show_header=False,pad_edge=False, show_edge=False, padding = (0,0), box=box.SIMPLE)
+        datatable = harpy_table()
         datatable.add_column("detail", justify="left", style="green", no_wrap=True)
         datatable.add_column("value", justify="left")
         datatable.add_row("Duration:", time_text)
         if self.summary:
             datatable.add_row("Summary: ", os.path.join(_relpath, "workflow", os.path.basename(self.summary)))
-        datatable.add_row("Workflow Log:", os.path.join(_relpath, "logs", "snakemake", os.path.basename(f"{self.snakemake_logfile}.gz")))
+        _smlog = last_sm_log(self.output_directory)
+        if _smlog:
+            datatable.add_row("Workflow Log:", os.path.join(_relpath, ".snakemake", 'log', _smlog))
         CONSOLE.rule("[bold]Workflow Finished[/] [default dim]" + _time.strftime('%d %b %Y @ %H:%M'), style="green")
         CONSOLE.print(datatable)
 
@@ -283,12 +271,10 @@ def launch(self, absolute:bool = False):
         cmd = self.snakemake_cmd_absolute if absolute else self.snakemake_cmd_relative
 
         try:
-            launch_snakemake(cmd, self.output_directory, self.snakemake_logfile, self.quiet)
+            launch_snakemake(cmd, self.output_directory, self.quiet)
         finally:
             with open(os.path.join(self.output_directory, "workflow", f"{self.name.replace('_','.')}.summary"), "w") as f_out: 
                 f_out.write(Summary(self.config).get_text())
             self.purge_empty_logs()
-            if os.path.exists(self.snakemake_logfile):
-                gzip_file(os.path.join(self.output_directory, self.snakemake_logfile))
 
         self.print_onsuccess()

harpy/snakefiles/align_bwa.smk

@@ -1,10 +1,6 @@
 import os
 import re
-import logging
 
-onstart:
-    logfile_handler = logger_manager._default_filehandler(config["snakemake"]["log"])
-    logger.addHandler(logfile_handler)
 wildcard_constraints:
     sample = r"[a-zA-Z0-9._-]+"
 

harpy/snakefiles/align_strobe.smk

@@ -1,10 +1,6 @@
 import os
 import re
-import logging
 
-onstart:
-    logfile_handler = logger_manager._default_filehandler(config["snakemake"]["log"])
-    logger.addHandler(logfile_handler)
 wildcard_constraints:
     sample = r"[a-zA-Z0-9._-]+"
 

harpy/snakefiles/assembly.smk

@@ -1,9 +1,5 @@
 import os
-import logging
 
-onstart:
-    logfile_handler = logger_manager._default_filehandler(config["snakemake"]["log"])
-    logger.addHandler(logfile_handler)
 
 FQ1 = config["inputs"]["fastq_r1"]
 FQ2 = config["inputs"]["fastq_r2"]

harpy/snakefiles/deconvolve.smk

@@ -1,10 +1,6 @@
 import os
 import re
-import logging
 
-onstart:
-    logfile_handler = logger_manager._default_filehandler(config["snakemake"]["log"])
-    logger.addHandler(logfile_handler)
 wildcard_constraints:
     sample = r"[a-zA-Z0-9._-]+"
 

harpy/snakefiles/demultiplex_meier2021.smk

@@ -1,9 +1,5 @@
 import os
-import logging
 
-onstart:
-    logfile_handler = logger_manager._default_filehandler(config["snakemake"]["log"])
-    logger.addHandler(logfile_handler)
 wildcard_constraints:
     sample = r"[a-zA-Z0-9._-]+",
     FR = r"[12]",