Skip to content

Commit 3091fc7

Browse files
mtcarlonemtcarlone
authored
Merge pull request #493 from brooklyn-data/feat/safe-walk-dict
Feat: Safely convert mappings (dictionaries) to JSON serializable objects
2 parents 3e4070b + ed6af18 commit 3091fc7

6 files changed

Lines changed: 140 additions & 40 deletions

File tree

‎integration_test_project/dbt_project.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ vars:
1919
env_vars: ["TEST_ENV_VAR_NUMBER", "TEST_ENV_VAR_EMPTY", "TEST_ENV_VAR_WITH_QUOTE"]
2020
dbt_vars: ["test_dbt_vars_1", "test_dbt_vars_2", "test_dbt_vars_3"]
2121
dbt_artifacts_exclude_all_results: true
22+
is_development: '{{ env_var("IS_DEVELOPMENT", false) | as_bool }}'
2223

2324
models:
2425
+persist_docs:

‎integration_test_project/models/microbatch.sql‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88

99
unique_key = 'transaction_id',
1010
partition_by = {
11-
'field': 'store_name',
12-
'data_type': 'text',
11+
'field': 'transaction_ts',
12+
'data_type': 'datetime',
13+
'granularity': 'day'
1314
}
1415
)
1516
}}

‎macros/_macros.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,46 @@ macros:
8989
- name: model
9090
type: Node
9191
description: A node object from the dbt graph
92+
- name: update_nested_dict
93+
description: |
94+
A macro that walks through a mapping object (dictionary) and recursively converts types to
95+
something that can be JSON serialized.
96+
arguments:
97+
- name: dictionary
98+
type: mapping
99+
description: A mapping object to walk through
100+
- name: safe_copy_mapping
101+
description: |
102+
A macro that safely copies a mapping object (dictionary), using the `update_nested_dict` macro to ensure that the types are JSON serializable.
103+
arguments:
104+
- name: dictionary
105+
type: mapping
106+
description: A mapping object to copy
107+
- name: type_handler
108+
description: |
109+
Handles the conversion of a value to a JSON serializable type. Notable manages our conversion
110+
from datetime to string.
111+
arguments:
112+
- name: val
113+
type: any
114+
description: A value to convert to a JSON serializable type
115+
- name: is_serializable
116+
description: |
117+
A macro that checks if a value is JSON serializable.
118+
arguments:
119+
- name: val
120+
type: any
121+
description: A value to check if it is JSON serializable
122+
- name: raise_equality_warning
123+
description: |
124+
A macro to check for the equality of a particular mapping object pre/post conversion.
125+
We expect some warnings to be thrown, but meant to alert the user in case there's some
126+
unexpected warnings. Logs can be searched for the '!!! SEE BELOW !!!' string to find the
127+
pre/post conversion values.
128+
arguments:
129+
- name: dictionary
130+
type: mapping
131+
description: A mapping object to validate
92132

93133
## UPLOAD INDIVIDUAL DATASETS ##
94134
- name: upload_exposures

‎macros/upload_individual_datasets/upload_invocations.sql‎

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,27 +16,12 @@
1616
{%- do invocation_args_dict.update({"warn_error_options": warn_error_options}) %}
1717
{% endif %}
1818
{% endif %}
19+
{% set converted_invocation_args_dict = dbt_artifacts.safe_copy_mapping(invocation_args_dict) %}
1920

20-
{% if "event_time_start" in invocation_args_dict %}
21-
{% if invocation_args_dict.event_time_start is not string %}
22-
{% do invocation_args_dict.update(
23-
{"event_time_start": invocation_args_dict.event_time_start.strftime(dbt_artifacts.get_strftime_format())}
24-
) %}
25-
{% endif%}
26-
{% endif %}
27-
{% if "event_time_end" in invocation_args_dict %}
28-
{% if invocation_args_dict.event_time_end is not string %}
29-
{% do invocation_args_dict.update(
30-
{"event_time_end": invocation_args_dict.event_time_end.strftime(dbt_artifacts.get_strftime_format())}
31-
) %}
32-
{% endif%}
33-
{% endif %}
34-
35-
{{ log(invocation_args_dict) }}
36-
{{ return(adapter.dispatch("get_invocations_dml_sql", "dbt_artifacts")()) }}
21+
{{ return(adapter.dispatch("get_invocations_dml_sql", "dbt_artifacts")(converted_invocation_args_dict)) }}
3722
{%- endmacro %}
3823

39-
{% macro default__get_invocations_dml_sql() -%}
24+
{% macro default__get_invocations_dml_sql(invocation_args=invocation_args_dict) -%}
4025
{% set invocation_values %}
4126
select
4227
{{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }},
@@ -97,7 +82,7 @@
9782
null, {# dbt_vars #}
9883
{% endif %}
9984

100-
'{{ tojson(invocation_args_dict) | replace('\\', '\\\\') | replace("'", "\\'") }}', {# invocation_args #}
85+
'{{ tojson(invocation_args) | replace('\\', '\\\\') | replace("'", "\\'") }}', {# invocation_args #}
10186

10287
{% set metadata_env = {} %}
10388
{% for key, value in dbt_metadata_envs.items() %}
@@ -111,7 +96,7 @@
11196

11297
{% endmacro -%}
11398

114-
{% macro bigquery__get_invocations_dml_sql() -%}
99+
{% macro bigquery__get_invocations_dml_sql(invocation_args=invocation_args_dict) -%}
115100
{% set invocation_values %}
116101
(
117102
'{{ invocation_id }}', {# command_invocation_id #}
@@ -151,16 +136,16 @@
151136
null, {# dbt_vars #}
152137
{% endif %}
153138

154-
{% if invocation_args_dict.vars %}
139+
{% if invocation_args.vars %}
155140
{# vars - different format for pre v1.5 (yaml vs list) #}
156-
{% if invocation_args_dict.vars is string %}
141+
{% if invocation_args.vars is string %}
157142
{# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #}
158-
{% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %}
159-
{% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %}
143+
{% set parsed_inv_args_vars = fromyaml(invocation_args.vars) %}
144+
{% do invocation_args.update({'vars': parsed_inv_args_vars}) %}
160145
{% endif %}
161146
{% endif %}
162147

163-
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(invocation_args_dict) | replace("'", "\\'")) }}, {# invocation_args #}
148+
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(tojson(invocation_args) | replace("'", "\\'")) }}, {# invocation_args #}
164149

165150
{% set metadata_env = {} %}
166151
{% for key, value in dbt_metadata_envs.items() %}
@@ -174,7 +159,7 @@
174159

175160
{% endmacro -%}
176161

177-
{% macro postgres__get_invocations_dml_sql() -%}
162+
{% macro postgres__get_invocations_dml_sql(invocation_args=invocation_args_dict) -%}
178163
{% set invocation_values %}
179164
(
180165
'{{ invocation_id }}', {# command_invocation_id #}
@@ -214,16 +199,16 @@
214199
null, {# dbt_vars #}
215200
{% endif %}
216201

217-
{% if invocation_args_dict.vars %}
202+
{% if invocation_args.vars %}
218203
{# vars - different format for pre v1.5 (yaml vs list) #}
219-
{% if invocation_args_dict.vars is string %}
204+
{% if invocation_args.vars is string %}
220205
{# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #}
221-
{% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %}
222-
{% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %}
206+
{% set parsed_inv_args_vars = fromyaml(invocation_args.vars) %}
207+
{% do invocation_args.update({'vars': parsed_inv_args_vars}) %}
223208
{% endif %}
224209
{% endif %}
225210

226-
$${{ tojson(invocation_args_dict) }}$$, {# invocation_args #}
211+
$${{ tojson(invocation_args) }}$$, {# invocation_args #}
227212

228213
{% set metadata_env = {} %}
229214
{% for key, value in dbt_metadata_envs.items() %}
@@ -236,7 +221,7 @@
236221

237222
{% endmacro -%}
238223

239-
{% macro trino__get_invocations_dml_sql() -%}
224+
{% macro trino__get_invocations_dml_sql(invocation_args=invocation_args_dict) -%}
240225
{% set invocation_values %}
241226
(
242227
'{{ invocation_id }}', {# command_invocation_id #}
@@ -298,7 +283,7 @@
298283
{% endmacro -%}
299284
300285
301-
{% macro sqlserver__get_invocations_dml_sql() -%}
286+
{% macro sqlserver__get_invocations_dml_sql(invocation_args=invocation_args_dict) -%}
302287
{% set invocation_values %}
303288
select
304289
"1",
@@ -356,7 +341,7 @@
356341
{% else %}
357342
null, {# dbt_vars #}
358343
{% endif %}
359-
'{{ tojson(invocation_args_dict) | replace("'", "''") }}', {# invocation_args #}
344+
'{{ tojson(invocation_args) | replace("'", "''") }}', {# invocation_args #}
360345
361346
{% set metadata_env = {} %}
362347
{% for key, value in dbt_metadata_envs.items() %}

‎macros/upload_individual_datasets/upload_models.sql‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@
2424
{{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(15)) }}
2525
from values
2626
{% for model in models -%}
27-
{% set model_copy = dbt_artifacts.copy_model(model) -%}
27+
{# {% set model_copy = dbt_artifacts.copy_model(model) -%} #}
28+
{% set model_copy = dbt_artifacts.safe_copy_mapping(model) -%}
2829
(
2930
'{{ invocation_id }}', {# command_invocation_id #}
3031
'{{ model_copy.unique_id }}', {# node_id #}
@@ -58,7 +59,8 @@
5859
{% if models != [] %}
5960
{% set model_values %}
6061
{% for model in models -%}
61-
{% set model_copy = dbt_artifacts.copy_model(model) -%}
62+
{# {% set model_copy = dbt_artifacts.copy_model(model) -%} #}
63+
{% set model_copy = dbt_artifacts.safe_copy_mapping(model) -%}
6264
(
6365
'{{ invocation_id }}', {# command_invocation_id #}
6466
'{{ model_copy.unique_id }}', {# node_id #}
@@ -92,7 +94,8 @@
9294
{% if models != [] %}
9395
{% set model_values %}
9496
{% for model in models -%}
95-
{% set model_copy = dbt_artifacts.copy_model(model) -%}
97+
{# {% set model_copy = dbt_artifacts.copy_model(model) -%} #}
98+
{% set model_copy = dbt_artifacts.safe_copy_mapping(model) -%}
9699
(
97100
'{{ invocation_id }}', {# command_invocation_id #}
98101
'{{ model_copy.unique_id }}', {# node_id #}
@@ -131,7 +134,8 @@
131134
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"
132135
from ( values
133136
{% for model in models -%}
134-
{% set model_copy = dbt_artifacts.copy_model(model) -%}
137+
{# {% set model_copy = dbt_artifacts.copy_model(model) -%} #}
138+
{% set model_copy = dbt_artifacts.safe_copy_mapping(model) -%}
135139
(
136140
'{{ invocation_id }}', {# command_invocation_id #}
137141
'{{ model_copy.unique_id }}', {# node_id #}

‎macros/utils/safe_mapping.sql‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{% macro update_nested_dict(dictionary) %}
2+
{% set updated_dict = dictionary.copy() %}
3+
{% for key, value in dictionary.items() %}
4+
{% if value is mapping %}
5+
{# Recursively update nested dictionaries #}
6+
{% set nested_update = dbt_artifacts.update_nested_dict(dictionary[key]) %}
7+
{% do updated_dict.update({key: dbt_artifacts.type_handler(nested_update)}) %}
8+
{% else %}
9+
{# Update the value dynamically #}
10+
{% do updated_dict.update({key: dbt_artifacts.type_handler(value)}) %}
11+
{% endif %}
12+
{% endfor %}
13+
{{ return(updated_dict) }}
14+
{% endmacro %}
15+
16+
{% macro safe_copy_mapping(dictionary) %}
17+
{{ dbt_artifacts.raise_equality_warning(dictionary) }}
18+
{{ return(dbt_artifacts.update_nested_dict(dictionary)) }}
19+
{% endmacro %}
20+
21+
{% macro type_handler(val) %}
22+
{% set _val = val %}
23+
{% if _val is none %}
24+
{{ return(_val) }}
25+
{% else %}
26+
{% if dbt_artifacts.is_serializable(_val) %}
27+
{{ return(_val) }}
28+
{% else %}
29+
{#- this is super wonky, because we're ASSUMING this works if it's not serializable -#}
30+
{#- there's a real scenario where we attempt this and it just goes sideways -#}
31+
{% if _val.strftime is not none %}
32+
{#- convert to string explicitly -#}
33+
{{ return(_val.strftime(dbt_artifacts.get_strftime_format()) ) }}
34+
{% else %}
35+
{#- just send it -#}
36+
{{ return(_val | as_text ) }}
37+
{% endif %}
38+
{% endif %}
39+
{% endif %}
40+
{% endmacro %}
41+
42+
{% macro is_serializable(val) %}
43+
{% if val is string or val is boolean or val is number %}
44+
{{ return(true) }}
45+
{% elif val is mapping %}
46+
{{ return(true) }}
47+
{% elif val is iterable %}
48+
{{ return(true) }}
49+
{% elif val is sequence%}
50+
{{ return(true) }}
51+
{% elif val is none %}
52+
{{ return(true) }}
53+
{% else %}
54+
{{ return(false) }}
55+
{% endif %}
56+
{% endmacro %}
57+
58+
{% macro raise_equality_warning(dictionary) %}
59+
{% set is_dev = var('is_development', false) | as_bool %}
60+
{% if is_dev %}
61+
{% if dictionary != dbt_artifacts.update_nested_dict(dictionary) %}
62+
{{ log("Caught on: " ~ dictionary.name, is_dev) }}
63+
{{ log("ORIGINAL !!! SEE BELOW !!!\n" ~ dictionary) }}
64+
{{ log("COPY !!! SEE BELOW !!!\n" ~ dbt_artifacts.update_nested_dict(dictionary)) }}
65+
66+
{{ exceptions.warn("Original mapping and copied mapping do not align! Please validate and disregard if expected.") }}
67+
{% endif %}
68+
{% endif %}
69+
{% endmacro %}

0 commit comments

Comments
 (0)