update

Author: edumgt

README.md

@@ -69,4 +69,5 @@ The code within this repository is licensed under the MIT License. For content a
 ## 1019 test
 ## 1021 docker-compose.yml 을 사용하는데, 오류 사항 체크하면서
 ## docker compose up zavod 으로 사용
-## https://chatgpt.com/c/68f649fa-5f9c-8320-a65e-d432d096a15c 의 내용 확인 필요
+## https://chatgpt.com/c/68f649fa-5f9c-8320-a65e-d432d096a15c 의 내용 확인 필요
+## server 에서 실행 시 docker compose up -d zavod 으로

docker-compose.yml

@@ -22,31 +22,49 @@ services:
       timeout: 5s
       retries: 5
     volumes:
-      - ./postgres-data:/var/lib/postgresql/data   # ✅ 로컬에 DB 저장
+      - ./postgres-data:/var/lib/postgresql/data
 
   zavod:
     image: ghcr.io/opensanctions/opensanctions:latest
-    # container_name: opensanctions-etl   # ⚠️ 충돌 방지를 위해 생략
     depends_on:
       db:
         condition: service_healthy
     env_file:
       - .env
     volumes:
-      - .:/opensanctions
+      - .:/opensanctions   # ✅ 로컬 윈도우 경로와 매핑됨
     networks:
       - appnet
     command: >
       bash -c "
-        echo '🚀 Scanning datasets (excluding underscore folders)...' &&
+        echo '🚀 Scanning datasets (excluding _ folders & GCS-dependent)...' &&
+        FAILED_FILE=/opensanctions/failed_datasets.md &&
+        echo '# ❌ Failed Datasets (Zavod Batch Run)' > $$FAILED_FILE &&
+        echo '' >> $$FAILED_FILE &&
+
         for file in $(find /opensanctions/datasets -type f -name '*.yml' ! -path '*/_*/*'); do
-          echo '⚙️  Running ETL for: '$$file &&
-          zavod crawl $$file &&
-          zavod export $$file &&
-          zavod load-db $$file &&
-          echo '✅ Completed: '$$file || echo '❌ Failed: '$$file;
+          # 내부 데이터(GCS) 의존성 검사
+          if grep -q 'fetch_internal_data' $$file; then
+            echo '⚠️  Skipping GCS-dependent dataset: '$$file;
+            continue;
+          fi
+
+          echo '⚙️  Running ETL for: '$$file;
+          (zavod crawl $$file && zavod export $$file && zavod load-db $$file);
+
+          if [ $$? -eq 0 ]; then
+            echo '✅ Completed: '$$file;
+          else
+            echo '❌ Failed: '$$file;
+            echo '- '$$file >> $$FAILED_FILE;
+          fi
         done &&
-        echo '🎉 All datasets processed.'
+
+        echo '' >> $$FAILED_FILE &&
+        echo '---' >> $$FAILED_FILE &&
+        echo '🕒 Run completed on: '$(date) >> $$FAILED_FILE &&
+
+        echo '🎉 All datasets processed. Failures logged in failed_datasets.md (host directory)'
       "
 
   web:

failed_datasets.md

@@ -0,0 +1,3 @@
+# ❌ Failed Datasets (Zavod Batch Run)
+
+- /opensanctions/datasets/am/hetq/am_hetq_officials.yml