remove from . import * in unittests

.github/workflows/build_and_test_epmt.yml

@@ -203,22 +203,6 @@ jobs:
           verbose: true
           use_pypi: true
 
-      - name: install pylint
-        run: |
-          pip install pylint
-
-      - name: run pylint on epmt module, ignoring ui (epmt-dash)
-        run: |
-          pylint --rcfile pylintrc --fail-under 6.5 --ignore-paths src/epmt/ui src/epmt
-
-      - name: run pylint on ui submodule (epmt-dash)
-        run: |
-          pylint --rcfile pylintrc --fail-under 5.5 src/epmt/ui
-
-      - name: run pylint on epmt entry-point script
-        run: |
-          pylint --rcfile pylintrc --fail-under 10 src/scripts/epmt
-
       - name: upload pip installable
         uses: actions/upload-artifact@v4
         with:
@@ -267,3 +251,19 @@ jobs:
       - name: epmt integration, kernel-compile
         run: |
           epmt -v integration 050-kernel-compile.bats
+
+      - name: install pylint
+        run: |
+          pip install pylint
+
+      - name: run pylint on epmt module, ignoring ui (epmt-dash)
+        run: |
+          pylint --rcfile pylintrc --fail-under 7.0 --ignore-paths src/epmt/ui src/epmt
+
+      - name: run pylint on ui submodule (epmt-dash)
+        run: |
+          pylint --rcfile pylintrc --fail-under 5.5 src/epmt/ui
+
+      - name: run pylint on epmt entry-point script
+        run: |
+          pylint --rcfile pylintrc --fail-under 10 src/scripts/epmt

src/epmt/epmt_query.py

@@ -1540,7 +1540,7 @@ def refmodel_set_active_metrics(ref_id, metrics):
     if unavailable_metrics:
         logger.warning(
             'Ignoring metrics that are not available in the trained model: {0}'.format(
-                unavailable__metrics ))
+                unavailable_metrics ))
     active_metrics = list(metrics_set & all_metrics)
     logger.info('Active metrics for model set to: %s', str(active_metrics))
     info_dict = dict.copy(r.info_dict or {})

src/epmt/test/__init__.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-from epmt.orm.sqlalchemy.models import Job, UnprocessedJob, Process
-from epmt.orm.sqlalchemy.general import orm_get, orm_dump_schema, orm_commit, orm_is_query
-from epmt.orm import db_session, setup_db, orm_db_provider, orm_in_memory, Operation
-from epmt.epmtlib import timing, get_install_root, get_username, str_dict
-from epmt.epmt_cmds import epmt_submit
-import epmt.epmt_query as eq
-import epmt.epmt_settings as settings
-import os
-import unittest
-from sys import stderr
-from glob import glob
-from os import environ, path
-from datetime import datetime
-from contextlib import nullcontext
-
-import logging
-from io import StringIO
-from json import loads
-
-import sqlite3
-import pandas as pd
-import numpy as np
-
-
-# ok, i kind of see what was happening here.
-# might take a bit to tease out but could be worth it for testing fidelity
-# i can remove this as the default import for many testing routines one at a time and slowly
-# add in the imports theyre actually testing.
-
-from epmt.epmtlib import epmt_logging_init, capture
-# ERROR
-epmt_logging_init(-1)
-# WARNING
-# epmt_logging_init(0)
-# INFO
-# epmt_logging_init(1)
-# DEBUG
-# epmt_logging_init(2)
-
-
-# this will be used repeatedly in the tests, so let's store it
-# in a variable instead of repeatedly calling the function
-install_root = get_install_root()

src/epmt/test/test_anysh.py

@@ -1,11 +1,13 @@
-#!/usr/bin/env python
+import os
+from os import environ
+import shutil
+import unittest
 
-from . import *
-# import os
+from epmt import epmt_settings as settings
+from epmt import epmt_query as eq
+from epmt.orm import setup_db
+from epmt.epmtlib import timing, capture
 
-# These will be used in both tests
-# One can embed them in the class, but referring to them with
-# a class prefix is ugly
 jobid = '1011'
 tuser = 'testuser'
 
@@ -16,7 +18,7 @@ def do_cleanup():
             os.remove(f)
         except OSError:
             pass
-    import shutil
+
     for d in ['/tmp/epmt']:
         try:
             shutil.rmtree('/tmp/epmt')
@@ -28,9 +30,8 @@ def do_cleanup():
 @timing
 def setUpModule():
     setup_db(settings)
-#    print('\n' + str(settings.db_params))
     do_cleanup()
-    from os import environ
+
     environ['SLURM_JOB_ID'] = jobid
     environ['SLURM_JOB_USER'] = tuser
     settings.post_process_job_on_ingest = True
@@ -51,13 +52,9 @@ def test_run_auto(self):
             self.assertEqual(0, results)
 
     def test_monolithic(self):
-        from epmt.epmt_cmds import epmt_check, epmt_source, epmt_start_job, epmt_dump_metadata, epmt_run, epmt_stop_job, epmt_stage, epmt_submit
+        from epmt.epmt_cmds import epmt_source, epmt_start_job, epmt_dump_metadata, epmt_run, epmt_stop_job, epmt_stage, epmt_submit
         with capture() as (out, err):
-            # Check will fail because of kernel paranoid, but we can't be
-            # sure it will always fail.
-            # TODO: Fix this so we can count on it to either always fail or always
-            # pass. Since we can't count on the kernel paranoid setting to be right,
-            # it might be better to set this up to always fail. But how?
+            # TODO see if this works 
             # results = epmt_check()
             # self.assertEqual(results, False)
 

src/epmt/test/test_cmds.py

@@ -1,16 +1,19 @@
-#!/usr/bin/env python
+from os import path
+from contextlib import nullcontext
+import unittest
+from glob import glob
 
-# the import below is crucial to get a sane test environment
-from . import *
+from epmt import epmt_query as eq, epmt_settings as settings
+from epmt.orm import db_session, orm_in_memory, setup_db, orm_db_provider
+from epmt.orm.sqlalchemy.models import UnprocessedJob
+from epmt.epmtlib import timing, capture, get_install_root, epmt_logging_init
+from epmt.epmt_cmds import epmt_dbsize, epmt_submit
 
-from epmt.epmt_cmds import epmt_dbsize
-from epmt.epmt_cmd_delete import epmt_delete_jobs
 from epmt.epmt_cmd_list import ( epmt_list_jobs, epmt_list_procs, epmt_list_job_proc_tags,
                                  epmt_list_refmodels, epmt_list_op_metrics, epmt_list_thread_metrics )
 from epmt.epmt_daemon import is_daemon_running, daemon_loop
 
-# from epmt.orm.sqlalchemy.models import UnprocessedJob
-# from os import path
+install_root=get_install_root()
 
 
 def do_cleanup():
@@ -124,7 +127,6 @@ def test_daemon_post_process(self):
         # and unprocessed jobs. Then we run the daemon loop once.
         # That should clear the backlog of unprocessed and
         # unanalyzed jobs
-        from epmt.epmt_job import post_process_pending_jobs
         self.assertTrue(is_daemon_running() == (False, -1))
 
         settings.post_process_job_on_ingest = False
@@ -267,7 +269,7 @@ def test_stage(self):
         epmt_logging_init(-1)
 
     def test_yyy_retire(self):
-        from datetime import datetime, timedelta
+        from datetime import datetime
         # with capture() as (out,err):
         #    epmt_submit(glob('{}/test/data/daemon/627919.tgz'.format(install_root)), dry_run=False)
         org_jobs = eq.get_jobs(["685000"], fmt='dict')

src/epmt/test/test_db_migration.py

@@ -1,11 +1,10 @@
-#!/usr/bin/env python
-
-# the import below is crucial to get a sane test environment
-from . import *
+import unittest
 
+from epmt import epmt_settings as settings
+from epmt.orm import setup_db, orm_in_memory
+from epmt.epmtlib import capture
 
 def setUpModule():
-    #    print('\n' + str(settings.db_params))
     setup_db(settings)
 
 

src/epmt/test/test_db_schema.py

@@ -1,18 +1,18 @@
-#!/usr/bin/env python
-
-# the import below is crucial to get a sane test environment
-from . import *
+import unittest
 
+from epmt import epmt_settings as settings
+from epmt.orm import setup_db, orm_db_provider, db_session
+from epmt.epmtlib import timing, capture
+from epmt.orm.sqlalchemy.models import Process
+from epmt.orm.sqlalchemy.general import orm_dump_schema, orm_get
 
 @timing
 def setUpModule():
-    #    print('\n' + str(settings.db_params))
     setup_db(settings)
 
 
 class EPMTDBSchema(unittest.TestCase):
 
-    # TODO: We need to make this test work for Pony as well
     def test_schema(self):
         with capture() as (out, err):
             retval = orm_dump_schema()
@@ -25,8 +25,8 @@ def test_schema(self):
     # Pony has a bug and only uses 32-bit integers for the PK
     # SQLite doesn't support the ALTER bigint migration. So
     # this test only works for SQLA+PostgreSQL
-    @unittest.skipUnless((settings.orm == 'sqlalchemy') and (orm_db_provider()
-                         == 'postgres'), 'only works with SQLAlchemy+PostgreSQL')
+    @unittest.skipUnless( (settings.orm == 'sqlalchemy') and (orm_db_provider() == 'postgres'),
+                          'only works with SQLAlchemy+PostgreSQL')
     @db_session
     def test_process_pk_bigint(self):
         pk_id = 4000000000

src/epmt/test/test_explore.py

@@ -1,17 +1,22 @@
-#!/usr/bin/env python
+from glob import glob
+import unittest
+
+from epmt import epmt_settings as settings
+from epmt import epmt_query as eq
+from epmt.orm import setup_db
+from epmt.epmt_cmds import epmt_submit
+from epmt.epmtlib import capture, timing, get_install_root
 
-# the import below is crucial to get a sane test environment
-from . import *
 import epmt.epmt_exp_explore as exp
 
+install_root = get_install_root()
 
 def do_cleanup():
     eq.delete_jobs(['685000', '685003'], force=True, remove_models=True)
 
 
 @timing
 def setUpModule():
-    print('\n' + str(settings.db_params))
     setup_db(settings)
     do_cleanup()
     datafiles = '{}/test/data/query/68500[03].tgz'.format(install_root)

src/epmt/test/test_lib.py

@@ -1,20 +1,15 @@
-#!/usr/bin/env python
+import unittest
+import logging
+from io import StringIO
 
-# the import below is crucial to get a sane test environment
-from . import *
-
-# import sqlite3
-# import pandas as pd
-# import numpy as np
-# import logging
-
-from epmt.epmtlib import dict_filter
-from epmt.epmtlib import merge_intervals
-from epmt.epmtlib import encode2ints, decode2strings
-from epmt.epmtlib import dframe_encode_features, dframe_decode_features
-from epmt.epmtlib import hash_strings
-from epmt.epmtlib import get_install_root
+import sqlite3
+import pandas as pd
+import numpy as np
 
+from epmt.epmtlib import ( dict_filter, merge_intervals,
+                           encode2ints, decode2strings,
+                           dframe_encode_features, dframe_decode_features,
+                           hash_strings, get_install_root )
 
 # The class below tests library functions
 class EPMTLib(unittest.TestCase):
@@ -84,7 +79,7 @@ def test_install_root(self):
 
     # BROKEN FIX TODO
     def test_logfn(self):
-        from epmt.epmtlib import logfn, epmt_logging_init, capture
+        from epmt.epmtlib import logfn, epmt_logging_init
 
         # enable debug logging and,
         # remove all handlers and add our StringIO handler

src/epmt/test/test_outliers.py

@@ -1,12 +1,17 @@
-#!/usr/bin/env python
-
-# the import below is crucial to get a sane test environment
-from . import *
-# from json import loads
+from json import loads, dumps
+from glob import glob
+import unittest
+from os import environ
+
+import epmt.epmt_settings as settings
+from epmt.orm import setup_db, db_session
+import epmt.epmt_query as eq
 import epmt.epmt_outliers as eod
-from epmt.epmtlib import frozen_dict
-from json import dumps
+from epmt.epmtlib import frozen_dict, timing, capture, get_install_root
+from epmt.epmt_cmds import epmt_submit
+from epmt.orm.sqlalchemy.models import Job
 
+install_root=get_install_root()
 
 def do_cleanup():
     eq.delete_jobs(['kern-6656-20190614-190245', 'kern-6656-20190614-191138',
@@ -16,7 +21,6 @@ def do_cleanup():
 
 @timing
 def setUpModule():
-    #    print('\n' + str(settings.db_params))
     setup_db(settings)
     do_cleanup()
     datafiles = '{}/test/data/outliers/*.tgz'.format(install_root)

src/epmt/test/test_query.py

@@ -1,18 +1,28 @@
-#!/usr/bin/env python
+import unittest
+from glob import glob
+from datetime import datetime
 
-# the import below is crucial to get a sane test environment
-from . import *
+import pandas as pd
+
+from epmt.orm.sqlalchemy.models import Job, Process
+from epmt.orm.sqlalchemy.general import orm_get, orm_commit, orm_is_query
+import epmt.epmt_settings as settings
+from epmt.epmt_cmds import epmt_submit
+from epmt.orm import db_session, setup_db, Operation
+from epmt.epmtlib import timing, capture, epmt_logging_init, get_install_root, str_dict
+import epmt.epmt_query as eq
+
+install_root=get_install_root()
 epmt_logging_init(0)
-JOBS_LIST = ['685016', '685003', '685000']
 
+JOBS_LIST = ['685016', '685003', '685000']
 
 def do_cleanup():
     eq.delete_jobs(JOBS_LIST, force=True, remove_models=True)
 
 
 @timing
 def setUpModule():
-    #    print('\n' + str(settings.db_params))
     setup_db(settings)
     do_cleanup()
     datafiles = '{}/test/data/query/*.tgz'.format(install_root)