Skip to content

Commit 0678e6c

Browse files
vladd-bitvladd-bit
committed
Deploy: cleaned mounts for nifi/nifi-registry etc.
1 parent 66a2a60 commit 0678e6c

File tree

2 files changed

+110
-98
lines changed

2 files changed

+110
-98
lines changed

‎deploy/services.dev.yml‎

Lines changed: 55 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,59 @@ x-nifi-common: &nifi-common
6161
networks:
6262
- cognet
6363

64+
x-nifi-volumes: &nifi-volumes
65+
# Drivers
66+
- ../nifi/drivers:/opt/nifi/drivers
67+
68+
# User overrides bundled in the image
69+
- ../nifi/user_scripts:/opt/nifi/user_scripts:rw
70+
- ../nifi/user_schemas:/opt/nifi/user_schemas:rw
71+
72+
# Python processors (NiFi 2.x)
73+
- ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw
74+
75+
# Security certificates
76+
- ../security:/security:ro
77+
78+
# Security helper scripts
79+
- ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro
80+
81+
# NiFi configuration
82+
- ../nifi/conf/:/opt/nifi/nifi-current/conf/:ro
83+
84+
# Ingest data directory
85+
- ./${NIFI_DATA_PATH:-../data/}:/data/:rw
86+
87+
# DB schemas
88+
- ../services/cogstack-db/:/opt/cogstack-db/:rw
89+
90+
# MedCAT models
91+
- ./${RES_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw
92+
93+
# NiFi repositories/state
94+
- nifi-vol-logs:/opt/nifi/nifi-current/logs
95+
- nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository
96+
- nifi-vol-database:/opt/nifi/nifi-current/database_repository
97+
- nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository
98+
- nifi-vol-content:/opt/nifi/nifi-current/content_repository
99+
- nifi-vol-state:/opt/nifi/nifi-current/state
100+
101+
# Flowfile error output
102+
- nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors
103+
104+
x-nifi-registry-volumes: &nifi-registry-volumes
105+
# Registry configuration
106+
- ../nifi/nifi-registry/:/opt/nifi-registry/nifi-registry-current/conf/:ro
107+
108+
# Security certificates
109+
- ../security:/security:ro
110+
111+
# Registry persistence
112+
- nifi-registry-vol-database:/opt/nifi-registry/nifi-registry-current/database
113+
- nifi-registry-vol-flow-storage:/opt/nifi-registry/nifi-registry-current/flow_storage
114+
- nifi-registry-vol-work:/opt/nifi-registry/nifi-registry-current/work
115+
- nifi-registry-vol-logs:/opt/nifi-registry/nifi-registry-current/logs
116+
64117
#---------------------------------------------------------------------------#
65118
# Used services #
66119
#---------------------------------------------------------------------------#
@@ -99,47 +152,7 @@ services:
99152
reservations:
100153
cpus: "${NIFI_DOCKER_CPU_MIN}"
101154
memory: "${NIFI_DOCKER_RAM}"
102-
volumes:
103-
# INFO: drivers folder
104-
- ../nifi/drivers:/opt/nifi/drivers
105-
106-
# INFO: if there are local changes, map these content from local host to container
107-
# (normally, these directories below are bundled with our NiFi image)
108-
# N.B. The container user may not have the permission to read these directories/files.
109-
- ../nifi/user_scripts:/opt/nifi/user_scripts:rw
110-
- ../nifi/user_schemas:/opt/nifi/user_schemas:rw
111-
112-
# this is a direct mapping to where we store the NiFi python processors as of NiFi 2.x.x
113-
- ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw
114-
115-
# INFO: uncomment below to map security certificates if need to secure NiFi endpoints
116-
- ../security:/security:ro
117-
118-
# Security credentials scripts
119-
- ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro
120-
121-
# # Nifi properties file:
122-
- ../nifi/conf/:/opt/nifi/nifi-current/conf/:ro
123-
124-
# this is where you should place data to be ingested, under the form of symbolic
125-
- ./${NIFI_DATA_PATH:-../data/}:/data/:rw
126-
127-
# DB-schemas, from the services folder
128-
- ../services/cogstack-db/:/opt/cogstack-db/:rw
129-
130-
# medcat models
131-
- ./${RES_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw
132-
133-
# rest of volumes to persist the state
134-
- nifi-vol-logs:/opt/nifi/nifi-current/logs
135-
- nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository
136-
- nifi-vol-database:/opt/nifi/nifi-current/database_repository
137-
- nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository
138-
- nifi-vol-content:/opt/nifi/nifi-current/content_repository
139-
- nifi-vol-state:/opt/nifi/nifi-current/state
140-
141-
# errors generated during data processing
142-
- nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors
155+
volumes: *nifi-volumes
143156

144157
# INFO : Uncomment the below line to generate your own USERNAME and PASSWORD,
145158
# a bit messy this way as you will need to copy the credentials back
@@ -184,14 +197,7 @@ services:
184197
reservations:
185198
cpus: "${NIFI_REGISTRY_DOCKER_CPU_MIN}"
186199
memory: "${NIFI_REGISTRY_DOCKER_RAM}"
187-
volumes:
188-
- ../nifi/nifi-registry/:/opt/nifi-registry/nifi-registry-current/conf/:ro
189-
- ../security:/security:ro
190-
191-
- nifi-registry-vol-database:/opt/nifi-registry/nifi-registry-current/database
192-
- nifi-registry-vol-flow-storage:/opt/nifi-registry/nifi-registry-current/flow_storage
193-
- nifi-registry-vol-work:/opt/nifi-registry/nifi-registry-current/work
194-
- nifi-registry-vol-logs:/opt/nifi-registry/nifi-registry-current/logs
200+
volumes: *nifi-registry-volumes
195201
extra_hosts: *common-hosts
196202
tty: true
197203
ports:

‎deploy/services.yml‎

Lines changed: 55 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,59 @@ x-nifi-common: &nifi-common
7272
networks:
7373
- cognet
7474

75+
x-nifi-volumes: &nifi-volumes
76+
# Drivers
77+
- ../nifi/drivers:/opt/nifi/drivers
78+
79+
# User overrides bundled in the image
80+
- ../nifi/user_scripts:/opt/nifi/user_scripts:rw
81+
- ../nifi/user_schemas:/opt/nifi/user_schemas:rw
82+
83+
# Python processors (NiFi 2.x)
84+
- ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw
85+
86+
# Security certificates
87+
- ../security:/security:ro
88+
89+
# Security helper scripts
90+
- ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro
91+
92+
# NiFi configuration
93+
- ../nifi/conf/:/opt/nifi/nifi-current/conf/:ro
94+
95+
# Ingest data directory
96+
- ./${NIFI_DATA_PATH:-../data/}:/data/:rw
97+
98+
# DB schemas
99+
- ../services/cogstack-db/:/opt/cogstack-db/:rw
100+
101+
# MedCAT models
102+
- ./${RES_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw
103+
104+
# NiFi repositories/state
105+
- nifi-vol-logs:/opt/nifi/nifi-current/logs
106+
- nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository
107+
- nifi-vol-database:/opt/nifi/nifi-current/database_repository
108+
- nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository
109+
- nifi-vol-content:/opt/nifi/nifi-current/content_repository
110+
- nifi-vol-state:/opt/nifi/nifi-current/state
111+
112+
# Flowfile error output
113+
- nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors
114+
115+
x-nifi-registry-volumes: &nifi-registry-volumes
116+
# Registry configuration
117+
- ../nifi/nifi-registry/:/opt/nifi-registry/nifi-registry-current/conf/:ro
118+
119+
# Security certificates
120+
- ../security:/security:ro
121+
122+
# Registry persistence
123+
- nifi-registry-vol-database:/opt/nifi-registry/nifi-registry-current/database
124+
- nifi-registry-vol-flow-storage:/opt/nifi-registry/nifi-registry-current/flow_storage
125+
- nifi-registry-vol-work:/opt/nifi-registry/nifi-registry-current/work
126+
- nifi-registry-vol-logs:/opt/nifi-registry/nifi-registry-current/logs
127+
75128
x-db-common: &db-common
76129
<<: *common-ulimits
77130
shm_size: ${DATABASE_DOCKER_SHM_SIZE:-"1g"}
@@ -489,47 +542,7 @@ services:
489542
reservations:
490543
cpus: "${NIFI_DOCKER_CPU_MIN}"
491544
memory: "${NIFI_DOCKER_RAM}"
492-
volumes:
493-
# INFO: drivers folder
494-
- ../nifi/drivers:/opt/nifi/drivers
495-
496-
# INFO: if there are local changes, map these content from local host to container
497-
# (normally, these directories below are bundled with our NiFi image)
498-
# N.B. The container user may not have the permission to read these directories/files.
499-
- ../nifi/user_scripts:/opt/nifi/user_scripts:rw
500-
- ../nifi/user_schemas:/opt/nifi/user_schemas:rw
501-
502-
# this is a direct mapping to where we store the NiFi python processors as of NiFi 2.x.x
503-
- ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw
504-
505-
# INFO: uncomment below to map security certificates if need to secure NiFi endpoints
506-
- ../security:/security:ro
507-
508-
# Security credentials scripts
509-
- ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro
510-
511-
# # Nifi properties file:
512-
- ../nifi/conf/:/opt/nifi/nifi-current/conf/:ro
513-
514-
# this is where you should place data to be ingested, under the form of symbolic
515-
- ./${NIFI_DATA_PATH:-../data/}:/data/:rw
516-
517-
# DB-schemas, from the services folder
518-
- ../services/cogstack-db/:/opt/cogstack-db/:rw
519-
520-
# medcat models
521-
- ./${RES_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw
522-
523-
# rest of volumes to persist the state
524-
- nifi-vol-logs:/opt/nifi/nifi-current/logs
525-
- nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository
526-
- nifi-vol-database:/opt/nifi/nifi-current/database_repository
527-
- nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository
528-
- nifi-vol-content:/opt/nifi/nifi-current/content_repository
529-
- nifi-vol-state:/opt/nifi/nifi-current/state
530-
531-
# errors generated during data processing
532-
- nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors
545+
volumes: *nifi-volumes
533546

534547
# INFO : Uncomment the below line to generate your own USERNAME and PASSWORD,
535548
# a bit messy this way as you will need to copy the credentials back
@@ -574,14 +587,7 @@ services:
574587
reservations:
575588
cpus: "${NIFI_REGISTRY_DOCKER_CPU_MIN}"
576589
memory: "${NIFI_REGISTRY_DOCKER_RAM}"
577-
volumes:
578-
- ../nifi/nifi-registry/:/opt/nifi-registry/nifi-registry-current/conf/:ro
579-
- ../security:/security:ro
580-
581-
- nifi-registry-vol-database:/opt/nifi-registry/nifi-registry-current/database
582-
- nifi-registry-vol-flow-storage:/opt/nifi-registry/nifi-registry-current/flow_storage
583-
- nifi-registry-vol-work:/opt/nifi-registry/nifi-registry-current/work
584-
- nifi-registry-vol-logs:/opt/nifi-registry/nifi-registry-current/logs
590+
volumes: *nifi-registry-volumes
585591
extra_hosts: *common-hosts
586592
tty: true
587593
ports:

0 commit comments

Comments
 (0)