From 16fd5553b3d6928b89a17fab2f45980024bcdec6 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Thu, 26 Feb 2026 17:19:56 -0800 Subject: [PATCH 01/11] Added a oracle/aida db creation from another repo. --- slac_db/aida.py | 101 ++++++++++++++++++++ slac_db/create/__init__.py | 8 ++ slac_db/create/lcls_elements_csv.py | 26 ++++++ slac_db/create/meme_names.py | 15 +++ slac_db/oracle.py | 138 ++++++++++++++++++++++++++++ 5 files changed, 288 insertions(+) create mode 100644 slac_db/aida.py create mode 100644 slac_db/create/__init__.py create mode 100644 slac_db/create/lcls_elements_csv.py create mode 100644 slac_db/create/meme_names.py create mode 100644 slac_db/oracle.py diff --git a/slac_db/aida.py b/slac_db/aida.py new file mode 100644 index 0000000..3c6abb9 --- /dev/null +++ b/slac_db/aida.py @@ -0,0 +1,101 @@ +import os +import slac_db.config +import sqlalchemy +import pykern.sql_db +import slac_db.oracle + +_meta = None + +def exists_address(device=None, address=None): + head = slac_db.oracle.get_address_header(device=device) + with _session() as s: + return set( + r["address"] for r in s.select( + sqlalchemy.select( + s.t.addresses.c["address"] + ).where( + s.t.addresses.c["address"] == address + ) + ) + ) + +def get_addresses(device=None): + head = slac_db.oracle.get_address_header(device=device) + with _session() as s: + cs_address = s.t.addresses.c["address"] + return set( + r["address"] for r in s.select( + sqlalchemy.select( + cs_address + ).where( + cs_address.like(f"{head}%") + ) + ) + ) + +def recreate(parser): + assert not _meta + assert parser.addresses + if os.path.exists(_aida_uri()): + os.remove(_aida_uri()) + _Inserter(parser) + + +def search_addresses(device=None, query=None): + head = slac_db.oracle.get_address_header(device=device) + with _session() as s: + cs_address = s.t.addresses.c["address"] + return set( + r["address"] for r in s.select( + sqlalchemy.select( + cs_address + ).where( + cs_address.like(f"{head}:{query}") + ) + ) + ) + +class _Inserter: + def __init__(self, parser): + self.counts = {"addresses": 0} + with _session() as s: + self._addresses(parser.addresses, s) + + def _addresses(self, addresses, session): + i = 0 + for a in addresses: + print(f"\r{i} / {len(addresses)}", end="") + session.insert("addresses", address=a) + i = i + 1 + +def _db_type_prefix(uri): + if not uri.startswith("sqlite"): + uri = 'sqlite:///' + uri + return uri + +def _init_db(uri=None): + global _meta + if uri is None: + uri = _aida_uri() + uri = _db_type_prefix(uri) + schema = { + "addresses": { + "address": "str 64 primary_key", + } + } + _meta = pykern.sql_db.Meta( + uri=uri, + schema=schema + ) + +def _aida_uri(): + uri = ( + slac_db.config.root_directory() + / 'slac_db' / 'package_data' / 'aida_pvs.db' + ) + return str(uri) + +def _session(): + if _meta is None: + _init_db() + return _meta.session() diff --git a/slac_db/create/__init__.py b/slac_db/create/__init__.py new file mode 100644 index 0000000..844c95e --- /dev/null +++ b/slac_db/create/__init__.py @@ -0,0 +1,8 @@ +import slac_db.create.meme_names +import slac_db.create.lcls_elements_csv + +def oracle_db(csv_source=None): + slac_db.create.lcls_elements_csv.to_oracle_db(csv_source) + +def aida_db(): + slac_db.create.meme_names.to_aida_db() diff --git a/slac_db/create/lcls_elements_csv.py b/slac_db/create/lcls_elements_csv.py new file mode 100644 index 0000000..b6bb79c --- /dev/null +++ b/slac_db/create/lcls_elements_csv.py @@ -0,0 +1,26 @@ +import csv +import slac_db.config +import slac_db.oracle + +def to_oracle_db(csv_source=None): + p = _Parser() + return slac_db.oracle.recreate(p, csv_source=csv_source) + +class _Parser(): + def __init__(self, csv_source=None): + if not csv_source: + csv_source = ( + slac_db.config.package_data() / "lcls_elements.csv" + ) + self.rows = {} + with open(csv_source, "r") as c: + reader = csv.reader(c) + self._parse_csv(reader) + + def _parse_csv(self, reader): + names = [r.lower() for r in next(reader)] + i = 0 + for row in reader: + values = [None if v == '' else v for v in row] + self.rows[i] = dict(zip(names, values)) + i += 1 diff --git a/slac_db/create/meme_names.py b/slac_db/create/meme_names.py new file mode 100644 index 0000000..c0debca --- /dev/null +++ b/slac_db/create/meme_names.py @@ -0,0 +1,15 @@ +import slac_db.aida + +def to_aida_db(): + return slac_db.aida.recreate(_Parser()) + +class _Parser: + def __init__(self): + self.addresses = set() + self._get_from_meme() + + def _get_from_meme(self): + import meme.names + address_list = meme.names.list_pvs("%", timeout=600) + for a in address_list: + self.addresses.add(a) diff --git a/slac_db/oracle.py b/slac_db/oracle.py new file mode 100644 index 0000000..2000b3d --- /dev/null +++ b/slac_db/oracle.py @@ -0,0 +1,138 @@ +import slac_db.config +import sqlalchemy +import pykern.sql_db +import os + + +_meta = None + +def get_address_header(device=None): + with _session() as s: + return s.select_one( + sqlalchemy.select( + s.t.elements.c["control system name"] + ).where( + s.t.elements.c["element"] == device + ) + )["control system name"] + +def get_devices(area=None, device_type=None): + if device_type is None: + device_type = "%" + with _session() as s: + return list( + r.element for r in s.select( + sqlalchemy.select( + s.t.elements.c["element"] + ).where( + s.t.elements.c["keyword"].like(device_type) + ).where( + s.t.elements.c["area"] == area + ) + ) + ) + +def get_device_row(element=None): + with _session() as s: + return s.select_one( + sqlalchemy.select( + s.t.elements + ).where( + s.t.elements.c["element"] == element + ) + ) + +def get_beampaths(): + def parse_beampaths(beampath_csv_row): + row = beampath_csv_row.replace(' ', '').split(',') + row = filter(None, row) + return row + + beampaths = set() + with _session() as s: + query = sqlalchemy.select(s.t.elements.c.beampath).distinct() + for r in s.select(query): + beampaths.update(parse_beampaths(r.beampath)) + return sorted(list(beampaths)) + + +def get_areas(): + def exclude_bad_patterns(column): + bad_patterns = ['\t- NO AREA -', '*%'] + filters = [None] * len(bad_patterns) + for i in range(0, len(bad_patterns)): + filters[i] = column.not_like(bad_patterns[i]) + return sqlalchemy.and_(*filters) + + with _session() as s: + return list( + r.area for r in s.select( + sqlalchemy.select( + s.t.elements.c.area + ).where( + exclude_bad_patterns(s.t.elements.c.area) + ).distinct() + ) + ) + +def recreate(parser): + assert not _meta + assert parser.rows + if os.path.exists(_oracle_uri()): + os.remove(_oracle_uri()) + _Inserter(parser) + + +class _Inserter(): + def __init__(self, parser): + with _session() as s: + self._rows(parser.rows, s) + + def _rows(self, rows, session): + i = 0 + for r in rows.values(): + ins = {} + for c in session.t.elements.c: + ins[c.name] = r[c.name] + session.insert("elements", **ins) + i = i + 1 + + +def _db_type_prefix(uri): + if not uri.startswith("sqlite"): + uri = 'sqlite:///' + uri + return uri + +def _init_db(uri=None): + global _meta + if uri is None: + uri = _oracle_uri() + uri = _db_type_prefix(uri) + schema = { + "elements": { + "Area": "str 64 nullable", + "Element": "str 64 primary_key", + "Control System Name": "str 64 nullable", + "Keyword": "str 64 nullable", + "Beampath": "str 64 nullable", + "SumL (m)": "float 64 nullable", + "Effective Length (m)": "float 64 nullable", + "Rf Frequency (MHz)": "float 64 nullable" + } + } + _meta = pykern.sql_db.Meta( + uri=uri, + schema=schema + ) + +def _oracle_uri(): + uri = ( + slac_db.config.root_directory() + / 'slac_db' / 'package_data' / 'lcls_elements.sqlite3' + ) + return str(uri) + +def _session(): + if _meta is None: + _init_db() + return _meta.session() From 290a28d004eb94f6ea5efd98ee29b7fa019dfa62 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Thu, 5 Mar 2026 14:52:59 -0800 Subject: [PATCH 02/11] merge with main --- .gitattributes | 7 +- pyproject.toml | 3 +- slac_db/aida.py | 3 +- slac_db/create/lcls_elements_csv.py | 4 +- slac_db/oracle.py | 3 +- tests/aidalist_test.py | 0 tests/test_aida.py | 12 + tests/test_data/DIAG0.yaml | 754 +++++++++++++++++++++++++ tests/test_data/example.db | Bin 0 -> 20480 bytes tests/test_data/expected_areas.csv | 1 + tests/test_data/expected_beampaths.csv | 1 + tests/test_oracle.py | 53 ++ 12 files changed, 830 insertions(+), 11 deletions(-) create mode 100644 tests/aidalist_test.py create mode 100644 tests/test_aida.py create mode 100644 tests/test_data/DIAG0.yaml create mode 100644 tests/test_data/example.db create mode 100644 tests/test_data/expected_areas.csv create mode 100644 tests/test_data/expected_beampaths.csv create mode 100644 tests/test_oracle.py diff --git a/.gitattributes b/.gitattributes index fd2328b..ef5e04f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,3 @@ -# slac_db/yaml/* linguist-generated -# slac_db/package_data/* linguist-generated -# tests/test_data/* linguist-generated -# tests/test_data/lcls-tools-yaml/* linguist-generated +slac_db/yaml/* linguist-generated +slac_db/package_data/* linguist-generated +tests/test_data/* linguist-generated diff --git a/pyproject.toml b/pyproject.toml index 71aeaac..84ccd1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,9 @@ classifiers = [ ] dependencies = [ "numpy", - "pyyaml", "pyepics", + "pykern", + "pyyaml", ] description = "Tools to support high level application development at LCLS using Python" dynamic = ["version"] diff --git a/slac_db/aida.py b/slac_db/aida.py index 3c6abb9..f874e9a 100644 --- a/slac_db/aida.py +++ b/slac_db/aida.py @@ -90,8 +90,7 @@ def _init_db(uri=None): def _aida_uri(): uri = ( - slac_db.config.root_directory() - / 'slac_db' / 'package_data' / 'aida_pvs.db' + slac_db.config.package_data() / 'aida_pvs.db' ) return str(uri) diff --git a/slac_db/create/lcls_elements_csv.py b/slac_db/create/lcls_elements_csv.py index b6bb79c..06e3bcd 100644 --- a/slac_db/create/lcls_elements_csv.py +++ b/slac_db/create/lcls_elements_csv.py @@ -3,8 +3,8 @@ import slac_db.oracle def to_oracle_db(csv_source=None): - p = _Parser() - return slac_db.oracle.recreate(p, csv_source=csv_source) + p = _Parser(csv_source=csv_source) + return slac_db.oracle.recreate(p) class _Parser(): def __init__(self, csv_source=None): diff --git a/slac_db/oracle.py b/slac_db/oracle.py index 2000b3d..a53cc28 100644 --- a/slac_db/oracle.py +++ b/slac_db/oracle.py @@ -127,8 +127,7 @@ def _init_db(uri=None): def _oracle_uri(): uri = ( - slac_db.config.root_directory() - / 'slac_db' / 'package_data' / 'lcls_elements.sqlite3' + slac_db.config.package_data() / 'lcls_elements.sqlite3' ) return str(uri) diff --git a/tests/aidalist_test.py b/tests/aidalist_test.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_aida.py b/tests/test_aida.py new file mode 100644 index 0000000..c6939e7 --- /dev/null +++ b/tests/test_aida.py @@ -0,0 +1,12 @@ +import unittest +import csv +from pathlib import Path +import slac_db.aida + +class TestAida(unittest.TestCase): + def test_get_otrdg02_pvs(self): + all_pvs = slac_db.aida.get_addresses( + device="OTRDG04", + ) + num_pvs = 2838 + self.assertEqual(len(all_pvs), num_pvs) diff --git a/tests/test_data/DIAG0.yaml b/tests/test_data/DIAG0.yaml new file mode 100644 index 0000000..1ca6683 --- /dev/null +++ b/tests/test_data/DIAG0.yaml @@ -0,0 +1,754 @@ +bpms: + BPMDG000?: + controls_information: + PVs: {} + control_name: BPMS:DIAG0:136 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 43.879 + type: BPM + BPMDG001: + controls_information: + PVs: + tmit: BPMS:DIAG0:190:TMIT + x: BPMS:DIAG0:190:X + y: BPMS:DIAG0:190:Y + control_name: BPMS:DIAG0:190 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 46.232 + type: BPM + BPMDG002: + controls_information: + PVs: + tmit: BPMS:DIAG0:210:TMIT + x: BPMS:DIAG0:210:X + y: BPMS:DIAG0:210:Y + control_name: BPMS:DIAG0:210 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 47.094 + type: BPM + BPMDG003: + controls_information: + PVs: + tmit: BPMS:DIAG0:230:TMIT + x: BPMS:DIAG0:230:X + y: BPMS:DIAG0:230:Y + control_name: BPMS:DIAG0:230 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 47.956 + type: BPM + BPMDG004: + controls_information: + PVs: + tmit: BPMS:DIAG0:270:TMIT + x: BPMS:DIAG0:270:X + y: BPMS:DIAG0:270:Y + control_name: BPMS:DIAG0:270 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 50.114 + type: BPM + BPMDG005: + controls_information: + PVs: + tmit: BPMS:DIAG0:285:TMIT + x: BPMS:DIAG0:285:X + y: BPMS:DIAG0:285:Y + control_name: BPMS:DIAG0:285 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 50.763 + type: BPM + BPMDG008: + controls_information: + PVs: + tmit: BPMS:DIAG0:370:TMIT + x: BPMS:DIAG0:370:X + y: BPMS:DIAG0:370:Y + control_name: BPMS:DIAG0:370 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 54.763 + type: BPM + BPMDG009: + controls_information: + PVs: + tmit: BPMS:DIAG0:390:TMIT + x: BPMS:DIAG0:390:X + y: BPMS:DIAG0:390:Y + control_name: BPMS:DIAG0:390 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 55.463 + type: BPM + BPMDG011: + controls_information: + PVs: + tmit: BPMS:DIAG0:470:TMIT + x: BPMS:DIAG0:470:X + y: BPMS:DIAG0:470:Y + control_name: BPMS:DIAG0:470 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 59.233 + type: BPM + BPMDG012: + controls_information: + PVs: + tmit: BPMS:DIAG0:520:TMIT + x: BPMS:DIAG0:520:X + y: BPMS:DIAG0:520:Y + control_name: BPMS:DIAG0:520 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 61.631 + type: BPM + BPMDG0RF: + controls_information: + PVs: + tmit: BPMS:DIAG0:330:TMIT + x: BPMS:DIAG0:330:X + y: BPMS:DIAG0:330:Y + control_name: BPMS:DIAG0:330 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 52.74 + type: BPM +magnets: + BKRDG0: + controls_information: + PVs: + bact: KICK:DIAG0:110:BACT + bcon: KICK:DIAG0:110:BCON + bctrl: KICK:DIAG0:110:BCTRL + bdes: KICK:DIAG0:110:BDES + bmax: KICK:DIAG0:110:BMAX + bmin: KICK:DIAG0:110:BMIN + ctrl: KICK:DIAG0:110:CTRL + control_name: KICK:DIAG0:110 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 1.0 + sum_l_meters: 42.73 + type: BEND + BLRDG0: + controls_information: + PVs: + bact: BEND:DIAG0:155:BACT + bcon: BEND:DIAG0:155:BCON + bctrl: BEND:DIAG0:155:BCTRL + bdes: BEND:DIAG0:155:BDES + bmax: BEND:DIAG0:155:BMAX + bmin: BEND:DIAG0:155:BMIN + ctrl: BEND:DIAG0:155:CTRL + control_name: BEND:DIAG0:155 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.401 + sum_l_meters: 44.73 + type: BEND + BXDG0: + controls_information: + PVs: + bact: BEND:DIAG0:260:BACT + bcon: BEND:DIAG0:260:BCON + bctrl: BEND:DIAG0:260:BCTRL + bdes: BEND:DIAG0:260:BDES + bmax: BEND:DIAG0:260:BMAX + bmin: BEND:DIAG0:260:BMIN + ctrl: BEND:DIAG0:260:CTRL + control_name: BEND:DIAG0:260 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.4 + sum_l_meters: 49.486 + type: BEND + BYDG0: + controls_information: + PVs: + bact: BEND:DIAG0:510:BACT + bcon: BEND:DIAG0:510:BCON + bctrl: BEND:DIAG0:510:BCTRL + bdes: BEND:DIAG0:510:BDES + bmax: BEND:DIAG0:510:BMAX + bmin: BEND:DIAG0:510:BMIN + ctrl: BEND:DIAG0:510:CTRL + control_name: BEND:DIAG0:510 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.543 + sum_l_meters: 60.964 + type: BEND + QDG001: + controls_information: + PVs: + bact: QUAD:DIAG0:190:BACT + bcon: QUAD:DIAG0:190:BCON + bctrl: QUAD:DIAG0:190:BCTRL + bdes: QUAD:DIAG0:190:BDES + bmax: QUAD:DIAG0:190:BMAX + bmin: QUAD:DIAG0:190:BMIN + ctrl: QUAD:DIAG0:190:CTRL + control_name: QUAD:DIAG0:190 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.197 + sum_l_meters: 46.232 + type: QUAD + QDG002: + controls_information: + PVs: + bact: QUAD:DIAG0:210:BACT + bcon: QUAD:DIAG0:210:BCON + bctrl: QUAD:DIAG0:210:BCTRL + bdes: QUAD:DIAG0:210:BDES + bmax: QUAD:DIAG0:210:BMAX + bmin: QUAD:DIAG0:210:BMIN + ctrl: QUAD:DIAG0:210:CTRL + control_name: QUAD:DIAG0:210 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.197 + sum_l_meters: 47.094 + type: QUAD + QDG003: + controls_information: + PVs: + bact: QUAD:DIAG0:230:BACT + bcon: QUAD:DIAG0:230:BCON + bctrl: QUAD:DIAG0:230:BCTRL + bdes: QUAD:DIAG0:230:BDES + bmax: QUAD:DIAG0:230:BMAX + bmin: QUAD:DIAG0:230:BMIN + ctrl: QUAD:DIAG0:230:CTRL + control_name: QUAD:DIAG0:230 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.197 + sum_l_meters: 47.956 + type: QUAD + QDG004: + controls_information: + PVs: + bact: QUAD:DIAG0:270:BACT + bcon: QUAD:DIAG0:270:BCON + bctrl: QUAD:DIAG0:270:BCTRL + bdes: QUAD:DIAG0:270:BDES + bmax: QUAD:DIAG0:270:BMAX + bmin: QUAD:DIAG0:270:BMIN + ctrl: QUAD:DIAG0:270:CTRL + control_name: QUAD:DIAG0:270 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 50.114 + type: QUAD + QDG005: + controls_information: + PVs: + bact: QUAD:DIAG0:285:BACT + bcon: QUAD:DIAG0:285:BCON + bctrl: QUAD:DIAG0:285:BCTRL + bdes: QUAD:DIAG0:285:BDES + bmax: QUAD:DIAG0:285:BMAX + bmin: QUAD:DIAG0:285:BMIN + ctrl: QUAD:DIAG0:285:CTRL + control_name: QUAD:DIAG0:285 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 50.763 + type: QUAD + QDG006: + controls_information: + PVs: + bact: QUAD:DIAG0:300:BACT + bcon: QUAD:DIAG0:300:BCON + bctrl: QUAD:DIAG0:300:BCTRL + bdes: QUAD:DIAG0:300:BDES + bmax: QUAD:DIAG0:300:BMAX + bmin: QUAD:DIAG0:300:BMIN + ctrl: QUAD:DIAG0:300:CTRL + control_name: QUAD:DIAG0:300 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 51.463 + type: QUAD + QDG007: + controls_information: + PVs: + bact: QUAD:DIAG0:360:BACT + bcon: QUAD:DIAG0:360:BCON + bctrl: QUAD:DIAG0:360:BCTRL + bdes: QUAD:DIAG0:360:BDES + bmax: QUAD:DIAG0:360:BMAX + bmin: QUAD:DIAG0:360:BMIN + ctrl: QUAD:DIAG0:360:CTRL + control_name: QUAD:DIAG0:360 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 54.063 + type: QUAD + QDG008: + controls_information: + PVs: + bact: QUAD:DIAG0:370:BACT + bcon: QUAD:DIAG0:370:BCON + bctrl: QUAD:DIAG0:370:BCTRL + bdes: QUAD:DIAG0:370:BDES + bmax: QUAD:DIAG0:370:BMAX + bmin: QUAD:DIAG0:370:BMIN + ctrl: QUAD:DIAG0:370:CTRL + control_name: QUAD:DIAG0:370 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 54.763 + type: QUAD + QDG009: + controls_information: + PVs: + bact: QUAD:DIAG0:390:BACT + bcon: QUAD:DIAG0:390:BCON + bctrl: QUAD:DIAG0:390:BCTRL + bdes: QUAD:DIAG0:390:BDES + bmax: QUAD:DIAG0:390:BMAX + bmin: QUAD:DIAG0:390:BMIN + ctrl: QUAD:DIAG0:390:CTRL + control_name: QUAD:DIAG0:390 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 55.463 + type: QUAD + QDG010: + controls_information: + PVs: + bact: QUAD:DIAG0:455:BACT + bcon: QUAD:DIAG0:455:BCON + bctrl: QUAD:DIAG0:455:BCTRL + bdes: QUAD:DIAG0:455:BDES + bmax: QUAD:DIAG0:455:BMAX + bmin: QUAD:DIAG0:455:BMIN + ctrl: QUAD:DIAG0:455:CTRL + control_name: QUAD:DIAG0:455 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 58.533 + type: QUAD + QDG011: + controls_information: + PVs: + bact: QUAD:DIAG0:470:BACT + bcon: QUAD:DIAG0:470:BCON + bctrl: QUAD:DIAG0:470:BCTRL + bdes: QUAD:DIAG0:470:BDES + bmax: QUAD:DIAG0:470:BMAX + bmin: QUAD:DIAG0:470:BMIN + ctrl: QUAD:DIAG0:470:CTRL + control_name: QUAD:DIAG0:470 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.108 + sum_l_meters: 59.233 + type: QUAD + XCDG001: + controls_information: + PVs: + bact: XCOR:DIAG0:178:BACT + bcon: XCOR:DIAG0:178:BCON + bctrl: XCOR:DIAG0:178:BCTRL + bdes: XCOR:DIAG0:178:BDES + bmax: XCOR:DIAG0:178:BMAX + bmin: XCOR:DIAG0:178:BMIN + ctrl: XCOR:DIAG0:178:CTRL + control_name: XCOR:DIAG0:178 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 45.883 + type: XCOR + XCDG002: + controls_information: + PVs: + bact: XCOR:DIAG0:218:BACT + bcon: XCOR:DIAG0:218:BCON + bctrl: XCOR:DIAG0:218:BCTRL + bdes: XCOR:DIAG0:218:BDES + bmax: XCOR:DIAG0:218:BMAX + bmin: XCOR:DIAG0:218:BMIN + ctrl: XCOR:DIAG0:218:CTRL + control_name: XCOR:DIAG0:218 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 47.607 + type: XCOR + XCDG003: + controls_information: + PVs: + bact: XCOR:DIAG0:280:BACT + bcon: XCOR:DIAG0:280:BCON + bctrl: XCOR:DIAG0:280:BCTRL + bdes: XCOR:DIAG0:280:BDES + bmax: XCOR:DIAG0:280:BMAX + bmin: XCOR:DIAG0:280:BMIN + ctrl: XCOR:DIAG0:280:CTRL + control_name: XCOR:DIAG0:280 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 50.439 + type: XCOR + XCDG005: + controls_information: + PVs: + bact: XCOR:DIAG0:290:BACT + bcon: XCOR:DIAG0:290:BCON + bctrl: XCOR:DIAG0:290:BCTRL + bdes: XCOR:DIAG0:290:BDES + bmax: XCOR:DIAG0:290:BMAX + bmin: XCOR:DIAG0:290:BMIN + ctrl: XCOR:DIAG0:290:CTRL + control_name: XCOR:DIAG0:290 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 51.113 + type: XCOR + XCDG008: + controls_information: + PVs: + bact: XCOR:DIAG0:380:BACT + bcon: XCOR:DIAG0:380:BCON + bctrl: XCOR:DIAG0:380:BCTRL + bdes: XCOR:DIAG0:380:BDES + bmax: XCOR:DIAG0:380:BMAX + bmin: XCOR:DIAG0:380:BMIN + ctrl: XCOR:DIAG0:380:CTRL + control_name: XCOR:DIAG0:380 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 55.113 + type: XCOR + XCDG010: + controls_information: + PVs: + bact: XCOR:DIAG0:460:BACT + bcon: XCOR:DIAG0:460:BCON + bctrl: XCOR:DIAG0:460:BCTRL + bdes: XCOR:DIAG0:460:BDES + bmax: XCOR:DIAG0:460:BMAX + bmin: XCOR:DIAG0:460:BMIN + ctrl: XCOR:DIAG0:460:CTRL + control_name: XCOR:DIAG0:460 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 58.883 + type: XCOR + YCDG001: + controls_information: + PVs: + bact: YCOR:DIAG0:199:BACT + bcon: YCOR:DIAG0:199:BCON + bctrl: YCOR:DIAG0:199:BCTRL + bdes: YCOR:DIAG0:199:BDES + bmax: YCOR:DIAG0:199:BMAX + bmin: YCOR:DIAG0:199:BMIN + ctrl: YCOR:DIAG0:199:CTRL + control_name: YCOR:DIAG0:199 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 46.745 + type: YCOR + YCDG002: + controls_information: + PVs: + bact: YCOR:DIAG0:247:BACT + bcon: YCOR:DIAG0:247:BCON + bctrl: YCOR:DIAG0:247:BCTRL + bdes: YCOR:DIAG0:247:BDES + bmax: YCOR:DIAG0:247:BMAX + bmin: YCOR:DIAG0:247:BMIN + ctrl: YCOR:DIAG0:247:CTRL + control_name: YCOR:DIAG0:247 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 48.832 + type: YCOR + YCDG003: + controls_information: + PVs: + bact: YCOR:DIAG0:280:BACT + bcon: YCOR:DIAG0:280:BCON + bctrl: YCOR:DIAG0:280:BCTRL + bdes: YCOR:DIAG0:280:BDES + bmax: YCOR:DIAG0:280:BMAX + bmin: YCOR:DIAG0:280:BMIN + ctrl: YCOR:DIAG0:280:CTRL + control_name: YCOR:DIAG0:280 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 50.439 + type: YCOR + YCDG005: + controls_information: + PVs: + bact: YCOR:DIAG0:290:BACT + bcon: YCOR:DIAG0:290:BCON + bctrl: YCOR:DIAG0:290:BCTRL + bdes: YCOR:DIAG0:290:BDES + bmax: YCOR:DIAG0:290:BMAX + bmin: YCOR:DIAG0:290:BMIN + ctrl: YCOR:DIAG0:290:CTRL + control_name: YCOR:DIAG0:290 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 51.113 + type: YCOR + YCDG008: + controls_information: + PVs: + bact: YCOR:DIAG0:380:BACT + bcon: YCOR:DIAG0:380:BCON + bctrl: YCOR:DIAG0:380:BCTRL + bdes: YCOR:DIAG0:380:BDES + bmax: YCOR:DIAG0:380:BMAX + bmin: YCOR:DIAG0:380:BMIN + ctrl: YCOR:DIAG0:380:CTRL + control_name: YCOR:DIAG0:380 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 55.113 + type: YCOR + YCDG010: + controls_information: + PVs: + bact: YCOR:DIAG0:460:BACT + bcon: YCOR:DIAG0:460:BCON + bctrl: YCOR:DIAG0:460:BCTRL + bdes: YCOR:DIAG0:460:BDES + bmax: YCOR:DIAG0:460:BMAX + bmin: YCOR:DIAG0:460:BMIN + ctrl: YCOR:DIAG0:460:CTRL + control_name: YCOR:DIAG0:460 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.0 + sum_l_meters: 58.883 + type: YCOR +screens: + OTRDG02: + controls_information: + PVs: + filter_1_control: OTRS:DIAG0:420:FLT1_CTRL + filter_1_status: OTRS:DIAG0:420:FLT1_STS + filter_2_control: OTRS:DIAG0:420:FLT2_CTRL + filter_2_status: OTRS:DIAG0:420:FLT2_STS + image: OTRS:DIAG0:420:Image:ArrayData + lamp_power: OTRS:DIAG0:420:TGT_LAMP_PWR + n_bits: OTRS:DIAG0:420:N_OF_BITS + n_col: OTRS:DIAG0:420:Image:ArraySize1_RBV + n_row: OTRS:DIAG0:420:Image:ArraySize0_RBV + orient_x: OTRS:DIAG0:420:X_ORIENT + orient_y: OTRS:DIAG0:420:Y_ORIENT + ref_rate: OTRS:DIAG0:420:ArrayRate_RBV + ref_rate_vme: OTRS:DIAG0:420:FRAME_RATE + resolution: OTRS:DIAG0:420:RESOLUTION + sys_type: OTRS:DIAG0:420:SYS_TYPE + target_control: OTRS:DIAG0:420:PNEUMATIC + target_status: OTRS:DIAG0:420:TGT_STS + control_name: OTRS:DIAG0:420 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 56.813 + type: PROF + OTRDG04: + controls_information: + PVs: + filter_1_control: OTRS:DIAG0:525:FLT1_CTRL + filter_1_status: OTRS:DIAG0:525:FLT1_STS + filter_2_control: OTRS:DIAG0:525:FLT2_CTRL + filter_2_status: OTRS:DIAG0:525:FLT2_STS + image: OTRS:DIAG0:525:Image:ArrayData + lamp_power: OTRS:DIAG0:525:TGT_LAMP_PWR + n_bits: OTRS:DIAG0:525:N_OF_BITS + n_col: OTRS:DIAG0:525:Image:ArraySize1_RBV + n_row: OTRS:DIAG0:525:Image:ArraySize0_RBV + orient_x: OTRS:DIAG0:525:X_ORIENT + orient_y: OTRS:DIAG0:525:Y_ORIENT + ref_rate: OTRS:DIAG0:525:ArrayRate_RBV + ref_rate_vme: OTRS:DIAG0:525:FRAME_RATE + resolution: OTRS:DIAG0:525:RESOLUTION + sys_type: OTRS:DIAG0:525:SYS_TYPE + target_control: OTRS:DIAG0:525:PNEUMATIC + target_status: OTRS:DIAG0:525:TGT_STS + control_name: OTRS:DIAG0:525 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + sum_l_meters: 61.871 + type: PROF +tcavs: + TCXDG0: + controls_information: + PVs: + amplitude: TCAV:DIAG0:11:AREQ + amplitude_fbenb: TCAV:DIAG0:11:AFBENB + amplitude_fbst: TCAV:DIAG0:11:AFBST + amplitude_wocho: TCAV:DIAG0:11:AMPL_W0CH0 + mode_config: TCAV:DIAG0:11:MODECFG + phase: TCAV:DIAG0:11:PREQ + phase_avgnt: TCAV:DIAG0:11:PACT_AVGNT + phase_fbenb: TCAV:DIAG0:11:PFBENB + phase_fbst: TCAV:DIAG0:11:PFBST + rf_enable: TCAV:DIAG0:11:RF_ENABLE + control_name: TCAV:DIAG0:11 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + l_eff: 0.508 + rf_freq: 2856.0 + sum_l_meters: 53.313 + type: LCAV +wires: + WSDG01: + controls_information: + PVs: + abort_scan: WIRE:DIAG0:424:MOTR.STOP + install_angle: WIRE:DIAG0:424:INSTALLANGLE + motor: WIRE:DIAG0:424:MOTR + motor_rbv: WIRE:DIAG0:424:MOTR.RBV + scan_pulses: WIRE:DIAG0:424:SCANPULSES + speed: WIRE:DIAG0:424:MOTR.VELO + speed_max: WIRE:DIAG0:424:MOTR.VMAX + speed_min: WIRE:DIAG0:424:MOTR.VBAS + start_scan: WIRE:DIAG0:424:STARTSCAN + u_size: WIRE:DIAG0:424:UWIRESIZE + u_wire_inner: WIRE:DIAG0:424:UWIREINNER + u_wire_outer: WIRE:DIAG0:424:UWIREOUTER + use_u_wire: WIRE:DIAG0:424:USEUWIRE + use_x_wire: WIRE:DIAG0:424:USEXWIRE + use_y_wire: WIRE:DIAG0:424:USEYWIRE + x_size: WIRE:DIAG0:424:XWIRESIZE + x_wire_inner: WIRE:DIAG0:424:XWIREINNER + x_wire_outer: WIRE:DIAG0:424:XWIREOUTER + y_size: WIRE:DIAG0:424:YWIRESIZE + y_wire_inner: WIRE:DIAG0:424:YWIREINNER + y_wire_outer: WIRE:DIAG0:424:YWIREOUTER + control_name: WIRE:DIAG0:424 + metadata: + area: DIAG0 + beam_path: + - SC_DIAG0 + bpms_after_wire: + - BPMS:DIAG0:470 + - BPMS:DIAG0:520 + bpms_before_wire: + - BPMS:DIAG0:190 + - BPMS:DIAG0:210 + - BPMS:DIAG0:230 + - BPMS:DIAG0:270 + - BPMS:DIAG0:285 + - BPMS:DIAG0:330 + - BPMS:DIAG0:370 + - BPMS:DIAG0:390 + detectors: + - SBLM01A:DIAG0 + - LBLM01A:HTR + - LBLM01B:HTR + sum_l_meters: 57.113 + type: WIRE diff --git a/tests/test_data/example.db b/tests/test_data/example.db new file mode 100644 index 0000000000000000000000000000000000000000..d16a0e5457640606bed8e8c74dac07e830acc907 GIT binary patch literal 20480 zcmeI(zi!h&9Ki8&9R3Abvs4vAmQERvB%9CeF|SXzBAI$iF5zTe$_HpkBJZG2WLd72klT1sC$6^0>RNhyRd zbiJpmscUQAZ|k-xReV`7MDNr073;ZZtzHQ$UcIV5%O(*=009ILKmY**5I_I{1Q0;r z-vzeY_t!()5NDGU^=0~TtZ#cq=|!2>{gE20Oijw^$Zft=x4!O;miy|O57uql__kA~ z@5ic1)U@qz5CusjlVC57WHY+akuoS$DwF74;^a2c~)pKN(+;@7dYn$=r z)4avB*=D?En(K7*PE6~Uu>M%Tbwfb_0R#|0009ILKmY**5I_KdTP5(sw2kIx1M^zP MYdtpYNArmL7as13UjP6A literal 0 HcmV?d00001 diff --git a/tests/test_data/expected_areas.csv b/tests/test_data/expected_areas.csv new file mode 100644 index 0000000..ef0fd7f --- /dev/null +++ b/tests/test_data/expected_areas.csv @@ -0,0 +1 @@ +GUNB,L0B,LR20,HTR,COL0,DIAG0,COL,L1B,BC1B,COL1,L2B,BC2B,EMIT2,L3B,EXT,DOG,SFTDMP,INJ,L0F,DL10,L1F,BC11_1,BC11_2,L2F,BYP,BC14_1,BC14E,BC14P,BC14_2,L3F_1,SCAV,L3F_2,BC20,FF20,COMMON,EXPT20,SPECT2,GUN,GSPEC,L0,DL1,GTL,SPEC,L1,LI21,BC1,L2,BC2,LI24,L3,LI27,LI28,SPH,SPD,SPS,DASEL,LI30,CLTH,SLTD,SLTH,SLTS,CLTS,BSY,BSYS,BSYH,BSYA,LTU,LTUH,LTUS,UNDH,UND,UNDS,DMPH,DMPS,DMPH_1,DUMP,DMPHH,SFTS,SFTH,HXTES,SXTES,S10-PO,S10-CO,S10-EL,LI20,ELECTR,POSITR,52LINE,LI19,LI29,ASTA,LI18,EIC,LI25,HXR,SXR,SXX,LI15 diff --git a/tests/test_data/expected_beampaths.csv b/tests/test_data/expected_beampaths.csv new file mode 100644 index 0000000..fc4cd9a --- /dev/null +++ b/tests/test_data/expected_beampaths.csv @@ -0,0 +1 @@ +CU_ALINE,CU_GSPEC,CU_HTXI,CU_HXR,CU_HXTES,CU_SFTH,CU_SPEC,CU_SXR,F2_ELEC,F2_POSI,F2_SCAV,SC_BSYD,SC_DASEL,SC_DIAG0,SC_HXR,SC_S2_X,SC_SFTS,SC_STMO,SC_STXI,SC_SXR \ No newline at end of file diff --git a/tests/test_oracle.py b/tests/test_oracle.py new file mode 100644 index 0000000..d7507b6 --- /dev/null +++ b/tests/test_oracle.py @@ -0,0 +1,53 @@ +import unittest +import csv +from pathlib import Path +import slac_db.oracle +import slac_db.aida + + +test_data_path = Path(__file__).parent / 'test_data' + +class TestExample(unittest.TestCase): + + def setUp(self): + p = test_data_path / 'example.db' + slac_db.oracle._init_db(str(p)) + + def test_get_example_areas(self): + expected_areas = ["AREA1", "AREA2", "AREA3"] + areas = slac_db.oracle.get_areas() + self.assertEqual(areas, expected_areas) + + def test_get_example_beampaths(self): + expected_areas = ["LINE1", "LINE2", "LINE3"] + areas = slac_db.oracle.get_beampaths() + self.assertEqual(areas, expected_areas) + +class TestOracle(unittest.TestCase): + + def setUp(self): + slac_db.oracle._init_db() + + def test_get_oracle_areas(self): + p = test_data_path / 'expected_areas.csv' + with open(str(p), 'r', newline='') as f: + reader = csv.reader(f) + expected_areas = next(reader) + areas = slac_db.oracle.get_areas() + self.assertEqual(areas, expected_areas) + + def test_get_oracle_beampaths(self): + p = test_data_path / 'expected_beampaths.csv' + with open(str(p), 'r', newline='') as f: + reader = csv.reader(f) + expected_beampaths = next(reader) + beampaths = slac_db.oracle.get_beampaths() + self.assertEqual(beampaths, expected_beampaths) + + def test_get_profile_monitors(self): + profile_monitors = slac_db.oracle.get_devices( + area="DIAG0", + device_type="PROF" + ) + expected_devices = ["OTRDG02", "OTRDG04"] + self.assertEqual(profile_monitors, expected_devices) From 670fc9563c8fd77582a0c73c9001c5080b9403ad Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:14:51 -0700 Subject: [PATCH 03/11] Added lfs tracking for sqlite3 databases. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index ef5e04f..c59ad05 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ slac_db/yaml/* linguist-generated slac_db/package_data/* linguist-generated tests/test_data/* linguist-generated +*.sqlite3 filter=lfs diff=lfs merge=lfs -text From 6e3a3cbbe4f1aef496e8dd2dffe36c571d06debe Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:25:03 -0700 Subject: [PATCH 04/11] Added sqlite dbs --- slac_db/package_data/aida_pvs.sqlite3 | 3 +++ slac_db/package_data/lcls_elements.sqlite3 | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 slac_db/package_data/aida_pvs.sqlite3 create mode 100644 slac_db/package_data/lcls_elements.sqlite3 diff --git a/slac_db/package_data/aida_pvs.sqlite3 b/slac_db/package_data/aida_pvs.sqlite3 new file mode 100644 index 0000000..8387c27 --- /dev/null +++ b/slac_db/package_data/aida_pvs.sqlite3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dea92bee30e1e0327eaf53ceb0b182737f6d474bbed502bd57e31f37f371672 +size 548225024 diff --git a/slac_db/package_data/lcls_elements.sqlite3 b/slac_db/package_data/lcls_elements.sqlite3 new file mode 100644 index 0000000..5e7c2c1 --- /dev/null +++ b/slac_db/package_data/lcls_elements.sqlite3 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61078084eb2d40e8d833d26ec4eeaa7402d2037a01a77a1f629498efbb85f3f7 +size 704512 From bac7f3a7bcb6e2ac9f3c06d343323c145697448b Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:41:48 -0700 Subject: [PATCH 05/11] Tests pass and now we use LFS. --- slac_db/aida.py | 9 ++++++--- slac_db/oracle.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/slac_db/aida.py b/slac_db/aida.py index f874e9a..8a101e3 100644 --- a/slac_db/aida.py +++ b/slac_db/aida.py @@ -62,11 +62,14 @@ def __init__(self, parser): self._addresses(parser.addresses, s) def _addresses(self, addresses, session): + # We have to do it this way unfortunately. + # Bulk insert is not faster. + n = len(addresses) i = 0 for a in addresses: - print(f"\r{i} / {len(addresses)}", end="") session.insert("addresses", address=a) - i = i + 1 + i += 1 + print("{i} / {n}", end='\r') def _db_type_prefix(uri): if not uri.startswith("sqlite"): @@ -90,7 +93,7 @@ def _init_db(uri=None): def _aida_uri(): uri = ( - slac_db.config.package_data() / 'aida_pvs.db' + slac_db.config.package_data() / 'aida_pvs.sqlite3' ) return str(uri) diff --git a/slac_db/oracle.py b/slac_db/oracle.py index a53cc28..c8ab58f 100644 --- a/slac_db/oracle.py +++ b/slac_db/oracle.py @@ -51,8 +51,10 @@ def parse_beampaths(beampath_csv_row): beampaths = set() with _session() as s: query = sqlalchemy.select(s.t.elements.c.beampath).distinct() - for r in s.select(query): - beampaths.update(parse_beampaths(r.beampath)) + for r in s.select(query): + if r.beampath is None: + continue + beampaths.update(parse_beampaths(r.beampath)) return sorted(list(beampaths)) From 71e6793f084f5f6d44a666e223dc6d0b44a58613 Mon Sep 17 00:00:00 2001 From: eloise-nebula <98988862+eloise-nebula@users.noreply.github.com> Date: Tue, 10 Mar 2026 09:14:18 -0700 Subject: [PATCH 06/11] Configure LFS in tests workflow checkout step --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5cfcca9..1feb082 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,6 +22,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + lfs: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -36,4 +38,4 @@ jobs: run: | echo -e '## Test results\n\n```' >> "$GITHUB_STEP_SUMMARY" python -m unittest discover -s tests -v 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - echo '```' >> "$GITHUB_STEP_SUMMARY" \ No newline at end of file + echo '```' >> "$GITHUB_STEP_SUMMARY" From 80497e1b72bfffebd56b2adf9081bb14555694ea Mon Sep 17 00:00:00 2001 From: eloise-nebula <98988862+eloise-nebula@users.noreply.github.com> Date: Tue, 10 Mar 2026 09:14:51 -0700 Subject: [PATCH 07/11] Fix indentation in tests.yml for lfs option --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1feb082..c35d527 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - lfs: true + lfs: true - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From ba013eb0410f76c0b6baef3ec6f94b0411a7dc01 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Tue, 17 Mar 2026 10:00:00 -0700 Subject: [PATCH 08/11] chkp --- slac_db/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slac_db/__init__.py b/slac_db/__init__.py index 696b588..abf8090 100644 --- a/slac_db/__init__.py +++ b/slac_db/__init__.py @@ -18,7 +18,7 @@ def _flatten(nested_list): return _flatten(nested_list[0]) + _flatten(nested_list[1:]) return nested_list[:1] + _flatten(nested_list[1:]) beampath_definition_file = os.path.join( - slac_db.package_data(), "beampaths.yaml" + slac_db.config.package_data(), "beampaths.yaml" ) with open(beampath_definition_file, "r") as file: beampath_definitions = yaml.safe_load(file) From 521530a5d5e303f321cc303723e6d998e5078252 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:20:12 -0700 Subject: [PATCH 09/11] checkpoint --- slac_db/aida.py | 51 +++++++++++++++++++---------------------------- slac_db/oracle.py | 26 +++++++++++++++++++++--- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/slac_db/aida.py b/slac_db/aida.py index 8a101e3..ec0346c 100644 --- a/slac_db/aida.py +++ b/slac_db/aida.py @@ -6,24 +6,19 @@ _meta = None -def exists_address(device=None, address=None): - head = slac_db.oracle.get_address_header(device=device) - with _session() as s: - return set( - r["address"] for r in s.select( - sqlalchemy.select( - s.t.addresses.c["address"] - ).where( - s.t.addresses.c["address"] == address - ) - ) - ) - def get_addresses(device=None): + """Get all addresses per device. + + Args: + device (str): MAD name of the device as found in Oracle. + + Returns: + tuple: Sorted address values. + """ head = slac_db.oracle.get_address_header(device=device) with _session() as s: cs_address = s.t.addresses.c["address"] - return set( + return tuple(sorted( r["address"] for r in s.select( sqlalchemy.select( cs_address @@ -31,31 +26,27 @@ def get_addresses(device=None): cs_address.like(f"{head}%") ) ) - ) + )) def recreate(parser): + """Rebuild the local aida sqlite3 database + only if it is not already loaded. + + Args: + parser: Container for column data. + """ assert not _meta assert parser.addresses if os.path.exists(_aida_uri()): os.remove(_aida_uri()) _Inserter(parser) - -def search_addresses(device=None, query=None): - head = slac_db.oracle.get_address_header(device=device) - with _session() as s: - cs_address = s.t.addresses.c["address"] - return set( - r["address"] for r in s.select( - sqlalchemy.select( - cs_address - ).where( - cs_address.like(f"{head}:{query}") - ) - ) - ) - class _Inserter: + """Creates a session and commits rows to the db. + + Functions: + _addresses: Inserts all addresses in parser.addresses. + """ def __init__(self, parser): self.counts = {"addresses": 0} with _session() as s: diff --git a/slac_db/oracle.py b/slac_db/oracle.py index c8ab58f..475a6e1 100644 --- a/slac_db/oracle.py +++ b/slac_db/oracle.py @@ -7,6 +7,14 @@ _meta = None def get_address_header(device=None): + """Get address header of a device. + + Args: + device (str): MAD name of the device as found in Oracle. + + Returns: + tuple: The address header. + """ with _session() as s: return s.select_one( sqlalchemy.select( @@ -17,22 +25,34 @@ def get_address_header(device=None): )["control system name"] def get_devices(area=None, device_type=None): + """Get devices of one type from an area. + + Args: + area (str): Name of the accelerator area. + device_type (str): Type of device as listed in Oracle. + + Returns: + tuple: Device names in Z order. + """ if device_type is None: device_type = "%" with _session() as s: - return list( + return tuple( r.element for r in s.select( sqlalchemy.select( s.t.elements.c["element"] ).where( - s.t.elements.c["keyword"].like(device_type) + s.t.elements.c["keyword"] == device_type ).where( s.t.elements.c["area"] == area - ) + ).order_by(s.t.elements.c["SumL (m)"]) ) ) def get_device_row(element=None): + """Get the full SQLite row for an element. + + """ with _session() as s: return s.select_one( sqlalchemy.select( From 6c81eebdac4b83d8fc184ac038578a67f79d6bb2 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Wed, 1 Apr 2026 13:01:43 -0700 Subject: [PATCH 10/11] Checkpoint --- slac_db/{aida.py => directory_service.py} | 18 ++++-- slac_db/oracle.py | 62 ++++++++++++++----- ....sqlite3 => directory_service_pvs.sqlite3} | 0 tests/test_aida.py | 4 +- tests/test_oracle.py | 4 +- 5 files changed, 63 insertions(+), 25 deletions(-) rename slac_db/{aida.py => directory_service.py} (80%) rename slac_db/package_data/{aida_pvs.sqlite3 => directory_service_pvs.sqlite3} (100%) diff --git a/slac_db/aida.py b/slac_db/directory_service.py similarity index 80% rename from slac_db/aida.py rename to slac_db/directory_service.py index ec0346c..8c8ac1d 100644 --- a/slac_db/aida.py +++ b/slac_db/directory_service.py @@ -1,4 +1,5 @@ import os +import os.path import slac_db.config import sqlalchemy import pykern.sql_db @@ -29,7 +30,7 @@ def get_addresses(device=None): )) def recreate(parser): - """Rebuild the local aida sqlite3 database + """Rebuild the local directory_service sqlite3 database only if it is not already loaded. Args: @@ -37,8 +38,8 @@ def recreate(parser): """ assert not _meta assert parser.addresses - if os.path.exists(_aida_uri()): - os.remove(_aida_uri()) + if os.path.exists(_directory_service_uri()): + os.remove(_directory_service_uri()) _Inserter(parser) class _Inserter: @@ -68,9 +69,14 @@ def _db_type_prefix(uri): return uri def _init_db(uri=None): + """Initializes pykern sqlalchemy wrapper. Initialization + occurs when a session is first created. + + _meta: wrapper that holds sqlalchemy metadata. + """ global _meta if uri is None: - uri = _aida_uri() + uri = _directory_service_uri() uri = _db_type_prefix(uri) schema = { "addresses": { @@ -82,9 +88,9 @@ def _init_db(uri=None): schema=schema ) -def _aida_uri(): +def _directory_service_uri(): uri = ( - slac_db.config.package_data() / 'aida_pvs.sqlite3' + slac_db.config.package_data() / 'directory_service_pvs.sqlite3' ) return str(uri) diff --git a/slac_db/oracle.py b/slac_db/oracle.py index 475a6e1..b9d82fe 100644 --- a/slac_db/oracle.py +++ b/slac_db/oracle.py @@ -1,6 +1,7 @@ import slac_db.config import sqlalchemy import pykern.sql_db +import os.path import os @@ -45,13 +46,17 @@ def get_devices(area=None, device_type=None): s.t.elements.c["keyword"] == device_type ).where( s.t.elements.c["area"] == area - ).order_by(s.t.elements.c["SumL (m)"]) + ).order_by(s.t.elements.c["suml (m)"]) ) ) def get_device_row(element=None): - """Get the full SQLite row for an element. + """Get the full row for an element. + Args: + element: name of the element to get. + Returns: + sql alchemy row: Row object with each column. """ with _session() as s: return s.select_one( @@ -63,22 +68,32 @@ def get_device_row(element=None): ) def get_beampaths(): - def parse_beampaths(beampath_csv_row): - row = beampath_csv_row.replace(' ', '').split(',') - row = filter(None, row) - return row + """Get all beampaths from Oracle. + Returns: + List of beampaths sorted alphabetically. + """ beampaths = set() + def parse_beampaths(beampath_csv): + if beampath_csv is None: + return + c = beampath_csv.replace(' ', '').split(',') + c = filter(None, c) + beampaths.update(c) + with _session() as s: query = sqlalchemy.select(s.t.elements.c.beampath).distinct() for r in s.select(query): - if r.beampath is None: - continue - beampaths.update(parse_beampaths(r.beampath)) + parse_beampaths(r.beampath) return sorted(list(beampaths)) def get_areas(): + """Get all areas from Oracle. + + Returns: + List of areas sorted alphabetically. + """ def exclude_bad_patterns(column): bad_patterns = ['\t- NO AREA -', '*%'] filters = [None] * len(bad_patterns) @@ -98,26 +113,38 @@ def exclude_bad_patterns(column): ) def recreate(parser): - assert not _meta - assert parser.rows + """Rebuilds the sqlite copy of Oracle. + Fails if a connection has already been made. + + Args: + Parser object with attribute 'rows' for row data. + """ + if _meta: + raise AssertionError( + "Database connnection already initialized. " + + "Restart Python interpreter." + ) + if not hasattr(parser, "rows"): + raise AssertionError( + "Parser is missing attribute 'rows'. " + ) if os.path.exists(_oracle_uri()): os.remove(_oracle_uri()) _Inserter(parser) -class _Inserter(): +class _Inserter: + """Inserts rows into sqllite database. + """ def __init__(self, parser): with _session() as s: self._rows(parser.rows, s) - def _rows(self, rows, session): - i = 0 for r in rows.values(): ins = {} for c in session.t.elements.c: ins[c.name] = r[c.name] session.insert("elements", **ins) - i = i + 1 def _db_type_prefix(uri): @@ -126,6 +153,11 @@ def _db_type_prefix(uri): return uri def _init_db(uri=None): + """Initializes pykern sqlalchemy wrapper. Initialization + occurs when a session is first created. + + _meta: wrapper that holds sqlalchemy metadata. + """ global _meta if uri is None: uri = _oracle_uri() diff --git a/slac_db/package_data/aida_pvs.sqlite3 b/slac_db/package_data/directory_service_pvs.sqlite3 similarity index 100% rename from slac_db/package_data/aida_pvs.sqlite3 rename to slac_db/package_data/directory_service_pvs.sqlite3 diff --git a/tests/test_aida.py b/tests/test_aida.py index c6939e7..57d3b16 100644 --- a/tests/test_aida.py +++ b/tests/test_aida.py @@ -1,11 +1,11 @@ import unittest import csv from pathlib import Path -import slac_db.aida +import slac_db.directory_service class TestAida(unittest.TestCase): def test_get_otrdg02_pvs(self): - all_pvs = slac_db.aida.get_addresses( + all_pvs = slac_db.directory_service.get_addresses( device="OTRDG04", ) num_pvs = 2838 diff --git a/tests/test_oracle.py b/tests/test_oracle.py index d7507b6..0d1263f 100644 --- a/tests/test_oracle.py +++ b/tests/test_oracle.py @@ -2,7 +2,7 @@ import csv from pathlib import Path import slac_db.oracle -import slac_db.aida +import slac_db.directory_service test_data_path = Path(__file__).parent / 'test_data' @@ -49,5 +49,5 @@ def test_get_profile_monitors(self): area="DIAG0", device_type="PROF" ) - expected_devices = ["OTRDG02", "OTRDG04"] + expected_devices = ("OTRDG02", "OTRDG04") self.assertEqual(profile_monitors, expected_devices) From 8d008f2fff148b5ccdfbf05eccd0504b27bc4318 Mon Sep 17 00:00:00 2001 From: EY <98988862+eloiseyang@users.noreply.github.com> Date: Wed, 1 Apr 2026 14:36:48 -0700 Subject: [PATCH 11/11] Renamed to directory service. --- slac_db/create/meme_names.py | 6 +++--- tests/aidalist_test.py | 0 tests/{test_aida.py => test_directory_service.py} | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 tests/aidalist_test.py rename tests/{test_aida.py => test_directory_service.py} (78%) diff --git a/slac_db/create/meme_names.py b/slac_db/create/meme_names.py index c0debca..35ba2b2 100644 --- a/slac_db/create/meme_names.py +++ b/slac_db/create/meme_names.py @@ -1,7 +1,7 @@ -import slac_db.aida +import slac_db.directory_service -def to_aida_db(): - return slac_db.aida.recreate(_Parser()) +def to_directory_service_db(): + return slac_db.directory_service.recreate(_Parser()) class _Parser: def __init__(self): diff --git a/tests/aidalist_test.py b/tests/aidalist_test.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_aida.py b/tests/test_directory_service.py similarity index 78% rename from tests/test_aida.py rename to tests/test_directory_service.py index 57d3b16..3891a08 100644 --- a/tests/test_aida.py +++ b/tests/test_directory_service.py @@ -1,9 +1,7 @@ import unittest -import csv -from pathlib import Path import slac_db.directory_service -class TestAida(unittest.TestCase): +class TestDirectoyrService(unittest.TestCase): def test_get_otrdg02_pvs(self): all_pvs = slac_db.directory_service.get_addresses( device="OTRDG04",