# %pip install domolibraryHow might I dynamically update a connector stored in GitHub?
import os
import domolibrary.client.DomoAuth as dmda
auth = dmda.DomoTokenAuth(
domo_access_token=os.environ["DOMO_ACCESS_TOKEN"],
domo_instance=os.environ['DOMO_INSTANCE'],
)
await auth.print_is_token()warning this token has not been validated by who_am_i, run get_auth_token first
🎉 token_auth token retrieved from domo-community ⚙️TrueSearch DomoDatacenter for dataset
- you could just use DomoDatset.get_by_id()
sample search criteria in DomoDatacenter
- just monitor network traffic
{
"entities": ["DATASET"],
"count": 0,
"offset": 0,
"combineResults": False,
"query": "*",
"queryProfile": "DATA_CENTER",
"filters": [
{
"filterType": "term",
"field": "dataprovidername_facet",
"value": "Snowflake Unload V2",
"name": "Snowflake Unload V2",
"not": False,
}
],
"facetValuesToInclude": None,
"facetValueLimit": 400,
"facetValueOffset": 0,
"includePhonetic": True,
"useEntities": True,
}{'entities': ['DATASET'],
'count': 0,
'offset': 0,
'combineResults': False,
'query': '*',
'queryProfile': 'DATA_CENTER',
'filters': [{'filterType': 'term',
'field': 'dataprovidername_facet',
'value': 'Snowflake Unload V2',
'name': 'Snowflake Unload V2',
'not': False}],
'facetValuesToInclude': None,
'facetValueLimit': 400,
'facetValueOffset': 0,
'includePhonetic': True,
'useEntities': True}search DomoDatacenter
- retrieve dataset
- retrieve stream_id
- streams handle how data is ingested into Domo
import domolibrary.classes.DomoDatacenter as dmdc
domo_datacenter = dmdc.DomoDatacenter(auth=auth)
additional_filters_ls = [
{
"filterType": "term",
"field": "dataprovidername_facet",
"value": "JSON",
}
]
domo_datasets = await domo_datacenter.search_datasets(
auth=auth, additional_filters_ls=additional_filters_ls
)
domo_datasets[0:5][DomoDataset(id='5f90901a-f0e6-451e-9bc9-09e23e96bd7f', display_type='json5', data_provider_type='json5', name='t1', description='', row_count=1, column_count=6, stream_id=1658, owner={'id': '1893952720', 'name': 'Jae Wilson1', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>),
DomoDataset(id='d72a9337-1e3d-4795-8e6b-3d0191dcc59c', display_type='json5', data_provider_type='json5', name='Alerts - Domo API', description='', row_count=17, column_count=39, stream_id=885, owner={'id': '68216396', 'name': 'Elliott Leonard', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>),
DomoDataset(id='5eb2e87e-8e5a-43b6-ae17-93c5eb0cd89a', display_type='json5', data_provider_type='json5', name='DomoStats| Card Metadata', description='', row_count=2, column_count=9, stream_id=1194, owner={'id': '68216396', 'name': 'Elliott Leonard', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>),
DomoDataset(id='e2b0054d-eb2c-4a4a-8b69-ca8a9962b2a0', display_type='json5', data_provider_type='json5', name='DomoStats - Subscriber Instances', description='', row_count=2, column_count=9, stream_id=1663, owner={'id': '68216396', 'name': 'Elliott Leonard', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>),
DomoDataset(id='e14b23ef-066a-4beb-b460-0b38bf2b1d84', display_type='json5', data_provider_type='json5', name='EntryPoint', description='', row_count=1, column_count=24, stream_id=1715, owner={'id': '1728973208', 'name': 'Peter Shull', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>)]dataset_id = "5f90901a-f0e6-451e-9bc9-09e23e96bd7f"
domo_dataset = next((ds for ds in domo_datasets if ds.id == dataset_id))
domo_dataset.__dict__{'auth': DomoTokenAuth(domo_instance='domo-community', token_name='token_auth', is_valid_token=True, url_manual_login='https://domo-community.domo.com/auth/index?domoManualLogin=true'),
'id': '5f90901a-f0e6-451e-9bc9-09e23e96bd7f',
'display_type': 'json5',
'data_provider_type': 'json5',
'name': 't1',
'description': '',
'row_count': 1,
'column_count': 6,
'stream_id': 1658,
'owner': {'id': '1893952720',
'name': 'Jae Wilson1',
'type': 'USER',
'group': False},
'formula': {},
'schema': DomoDataset_Schema(dataset=DomoDataset(id='5f90901a-f0e6-451e-9bc9-09e23e96bd7f', display_type='json5', data_provider_type='json5', name='t1', description='', row_count=1, column_count=6, stream_id=1658, owner={'id': '1893952720', 'name': 'Jae Wilson1', 'type': 'USER', 'group': False}, formula={}, schema=..., tags=DomoDataset_Tags(dataset=..., tag_ls=[]), certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>), columns=[]),
'tags': DomoDataset_Tags(dataset=DomoDataset(id='5f90901a-f0e6-451e-9bc9-09e23e96bd7f', display_type='json5', data_provider_type='json5', name='t1', description='', row_count=1, column_count=6, stream_id=1658, owner={'id': '1893952720', 'name': 'Jae Wilson1', 'type': 'USER', 'group': False}, formula={}, schema=DomoDataset_Schema(dataset=..., columns=[]), tags=..., certification=None, PDP=<domolibrary.classes.DomoPDP.Dataset_PDP_Policies object>), tag_ls=[]),
'certification': None,
'PDP': <domolibrary.classes.DomoPDP.Dataset_PDP_Policies>}Update Stream
- get the stream
- extract the configuration information
- update config settings with new SQL
- push updated stream config to dataset
import domolibrary.routes.stream as stream_routes
stream_id = domo_dataset.stream_id
res = await stream_routes.get_stream_by_id(auth=auth, stream_id=stream_id)
stream = res.response
streamadjusting num_stacks_to_drop, consider revising `get_traceback` call
{'stack_length': 16, 'module_index': 12, 'num_stacks_to_drop_passed': 3}{'id': 1658,
'valid': True,
'invalidExecutionId': None,
'transport': {'type': 'CONNECTOR',
'description': 'com.domo.connector.json.customparsing',
'version': '4'},
'updateMethod': 'REPLACE',
'dataProvider': {'id': 3251,
'key': 'json5',
'name': 'JSON',
'url': None,
'defaultConnectorId': None,
'defaultConnectorLabel': None,
'authenticationScheme': 'fields',
'connectorValidatorPresent': False,
'resourceBundlePresent': False,
'moduleHandler': None,
'iconPicker': False,
'unsafe': False,
'scope': '0',
'producer': 'DOMO',
'version': '0',
'dateCreated': 1601579548000,
'dateUpdated': 1693235941000,
'visibility': 'WITH_CONNECTOR',
'type': 'STANDARD',
'customerOAuthConfigurationDataProviderType': None,
'authenticationSchemeConfiguration': [{'id': 12114,
'name': 'authentication',
'type': 'dropdown',
'text': 'Authentication Type',
'tooltipText': 'Authentication Type',
'required': False,
'encrypted': False,
'defaultValue': 'basic',
'restrictions': None,
'sortOrder': 1,
'data': {'options': [{'label': 'None', 'value': 'none'},
{'label': 'Basic Authentication or SFTP', 'value': 'basic'},
{'label': 'API key', 'value': 'apiKey'}]},
'configField': False},
{'id': 12115,
'name': 'username',
'type': 'string',
'text': 'Username',
'tooltipText': 'Enter a username if using SFTP or Basic Authentication, otherwise leave blank.',
'required': False,
'encrypted': False,
'defaultValue': '',
'restrictions': None,
'sortOrder': 2,
'data': {},
'configField': False},
{'id': 12116,
'name': 'password',
'type': 'password',
'text': 'Password',
'tooltipText': 'Enter a password if using SFTP or Basic Authentication, otherwise leave blank.',
'required': False,
'encrypted': True,
'defaultValue': '',
'restrictions': None,
'sortOrder': 3,
'data': {},
'configField': False},
{'id': 12117,
'name': 'apiType',
'type': 'dropdown',
'text': 'API key type',
'tooltipText': 'API key type',
'required': False,
'encrypted': False,
'defaultValue': 'header',
'restrictions': None,
'sortOrder': 4,
'data': {'options': [{'label': 'Pass as a header', 'value': 'header'},
{'label': 'Pass as a parameter', 'value': 'parameter'}]},
'configField': False},
{'id': 12118,
'name': 'name',
'type': 'string',
'text': 'Parameter/header Name',
'tooltipText': 'Parameter or header name',
'required': False,
'encrypted': False,
'defaultValue': '',
'restrictions': None,
'sortOrder': 5,
'data': {},
'configField': False},
{'id': 12119,
'name': 'apiKey',
'type': 'string',
'text': 'Api Key',
'tooltipText': 'Api Key',
'required': False,
'encrypted': True,
'defaultValue': '',
'restrictions': None,
'sortOrder': 6,
'data': {},
'configField': False}]},
'account': {'id': 91,
'userId': 1893952720,
'name': 'JSON Account',
'displayName': 'JSON Account',
'type': 'data',
'valid': True,
'dataProviderType': 'json5',
'credentialsType': 'fields',
'createdAt': 1707328700000,
'createdBy': 1893952720,
'modifiedAt': 1707328700000,
'modifiedBy': 1893952720,
'configurations': {},
'accountId': 91,
'accountTemplateId': None,
'accountTemplateAuthorizationId': None},
'accountTemplate': None,
'dataSource': {'id': '5f90901a-f0e6-451e-9bc9-09e23e96bd7f',
'displayType': 'json5',
'dataProviderType': 'json5',
'type': 'json5',
'name': 't1',
'description': '',
'owner': {'id': '1893952720',
'name': 'Jae Wilson1',
'type': 'USER',
'group': False},
'status': 'SUCCESS',
'created': 1713471499000,
'lastTouched': 1717526845000,
'lastUpdated': 1717447031461,
'nextUpdate': 1717533419597,
'rowCount': 1,
'columnCount': 6,
'cardInfo': {'cardCount': 0, 'cardViewCount': 0},
'properties': {'formulas': {'formulas': {}}},
'state': 'SUCCESS',
'validConfiguration': True,
'validAccount': True,
'accountId': 91,
'streamId': 1658,
'transportType': 'CONNECTOR',
'adc': False,
'adcExternal': False,
'cloudId': 'domo',
'cloudName': 'Domo',
'permissions': 'READ_WRITE_DELETE_SHARE_ADMIN',
'hidden': False,
'scheduleActive': True,
'cloudEngine': 'domo'},
'schemaDefinition': {'columns': [{'type': 'LONG',
'name': 'id',
'id': 'id',
'visible': True,
'metadata': {'colLabel': 'id', 'colFormat': '', 'isEncrypted': False}},
{'type': 'STRING',
'name': 'completed',
'id': 'completed',
'visible': True,
'metadata': {'colLabel': 'completed',
'colFormat': '',
'isEncrypted': False}},
{'type': 'STRING',
'name': 'title',
'id': 'title',
'visible': True,
'metadata': {'colLabel': 'title', 'colFormat': '', 'isEncrypted': False}},
{'type': 'LONG',
'name': 'userId',
'id': 'userId',
'visible': True,
'metadata': {'colLabel': 'userId', 'colFormat': '', 'isEncrypted': False}},
{'type': 'DOUBLE',
'name': '_BATCH_ID_',
'id': '_BATCH_ID_',
'visible': True,
'metadata': {'colLabel': '_BATCH_ID_',
'colFormat': '',
'isEncrypted': False}},
{'type': 'DATETIME',
'name': '_BATCH_LAST_RUN_',
'id': '_BATCH_LAST_RUN_',
'visible': True,
'metadata': {'colLabel': '_BATCH_LAST_RUN_',
'colFormat': '',
'isEncrypted': False}}]},
'scheduleExpression': '0 37 20 * * ? TZ=UTC',
'scheduleStartDate': None,
'advancedScheduleJson': '{"type":"DAY","at":"08:37 PM","timezone":"UTC"}',
'scheduleRetryExpression': None,
'scheduleRetryCount': 0,
'lastExecution': {'streamId': 1658,
'executionId': 49,
'toe': 'PN218JHC9K-NF07D-UM72F',
'startedAt': 1717447021.0,
'endedAt': 1717447031.0,
'updateMethod': 'REPLACE',
'index': True,
'retryCount': 0,
'retryExecution': None,
'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312',
'uploadId': 49,
'indexRequestKey': 20240603203710.594,
'currentState': 'SUCCESS',
'runType': 'AUTOMATED',
'createdAt': 1717447021.0,
'modifiedAt': 1717447031.0,
'latestPhase': None,
'currentPhase': None,
'removed': False,
'rowsInserted': 1,
'bytesInserted': 63,
'startedBy': None,
'cancelledBy': None,
'dataTag': None,
'peakMemoryUsedBytes': None,
'peakMemoryCommittedBytes': None,
'exportable': False,
'manualIndex': False,
'errors': []},
'lastSuccessfulExecution': {'streamId': 1658,
'executionId': 49,
'toe': 'PN218JHC9K-NF07D-UM72F',
'startedAt': 1717447021.0,
'endedAt': 1717447031.0,
'updateMethod': 'REPLACE',
'index': True,
'retryCount': 0,
'retryExecution': None,
'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312',
'uploadId': 49,
'indexRequestKey': 20240603203710.594,
'currentState': 'SUCCESS',
'runType': 'AUTOMATED',
'createdAt': 1717447021.0,
'modifiedAt': 1717447031.0,
'latestPhase': None,
'currentPhase': None,
'removed': False,
'rowsInserted': 1,
'bytesInserted': 63,
'startedBy': None,
'cancelledBy': None,
'dataTag': None,
'peakMemoryUsedBytes': None,
'peakMemoryCommittedBytes': None,
'exportable': False,
'manualIndex': False,
'errors': []},
'currentExecution': None,
'currentExecutionState': 'SUCCESS',
'createdAt': 1713471499.0,
'createdBy': 1893952720,
'modifiedAt': 1717509420.0,
'modifiedBy': 1893952720,
'scheduleState': 'ACTIVE',
'scheduleAssertion': False,
'inactiveScheduleCode': None,
'configuration': [{'streamId': 1658,
'category': 'METADATA',
'name': '_description_',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'retry.retryNumber',
'type': 'string',
'value': '0'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mergeChoice',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonParsing',
'type': 'string',
'value': '{"parsing":""}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'cloud',
'type': 'string',
'value': 'domo'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'schema',
'type': 'string',
'value': '{"automatedSchema":true}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonPaging',
'type': 'string',
'value': '{"baseUrl":"https://jsonplaceholder.typicode.com/todos/3","tokenType":"path","pagingType":"none","parsing":"","total":"true","pageLimitValue":0,"startPage":"1","encodeParameterValue":true,"encodeParameterKey":true,"pageParameter":"","tokenLocation":"first"}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mode',
'type': 'string',
'value': 'REPLACE'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.upsertKeys',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonSelection',
'type': 'string',
'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}]}stream_configuration = stream["configuration"]
stream_configuration[{'streamId': 1658,
'category': 'METADATA',
'name': '_description_',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'retry.retryNumber',
'type': 'string',
'value': '0'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mergeChoice',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonParsing',
'type': 'string',
'value': '{"parsing":""}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'cloud',
'type': 'string',
'value': 'domo'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'schema',
'type': 'string',
'value': '{"automatedSchema":true}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonPaging',
'type': 'string',
'value': '{"baseUrl":"https://jsonplaceholder.typicode.com/todos/3","tokenType":"path","pagingType":"none","parsing":"","total":"true","pageLimitValue":0,"startPage":"1","encodeParameterValue":true,"encodeParameterKey":true,"pageParameter":"","tokenLocation":"first"}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mode',
'type': 'string',
'value': 'REPLACE'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.upsertKeys',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonSelection',
'type': 'string',
'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}]original_obj = None
for index, obj in enumerate(stream_configuration):
if obj["category"] == "METADATA" and obj["name"] == "jsonSelection":
stream_configuration.pop(index)
original_obj = obj
print(
f"""
original_obj {original_obj}
position in config {index}"""
)
original_obj {'streamId': 1658, 'category': 'METADATA', 'name': 'jsonSelection', 'type': 'string', 'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}
position in config 9# the original config object doesn't store the "value" as json, instead it's stored as a string
# convert to json for manipulation
import json
new_value = json.loads(original_obj["value"])
new_value{'connectionMethod': 'Advanced',
'jsonUrl': 'https://jsonplaceholder.typicode.com/todos/8',
'httpMethod': 'GET',
'jsonLineReader': 'false',
'escapeBackslash': 'false',
'httpsHeaders': [],
'body': '',
'queryParameters': [],
'useDateFilter': False,
'dateSelection': {'dateType': 'date_range',
'startDate': {'type': 'relative', 'offset': 0},
'endDate': {'type': 'relative', 'offset': 0}},
'dateQueryParamsFormat': 'yyyy-MM-dd',
'certificateInputType': 'NoCertificate',
'encodeParameterKey': True,
'encodeParameterValue': True,
'datasetType': 'static',
'dateParameter': {'dateType': 'single_date',
'dateFrom': 'relative',
'dateFromOffset': 1,
'dateTo': 'relative',
'dateToOffset': 0,
'date': 'relative',
'dateOffset': 1},
'useBody': False,
'dynamicValuesType': 'enter',
'rateLimitSelection': 'false',
'timeUnit': 'second',
'dateParameterType': 'separate'}# read in the new parameter value
new_query_path = "./new_query.txt"
assert os.path.exists(new_query_path)
with open(new_query_path, "r", encoding="utf-8") as f:
new_query = f.read()
new_query'https://jsonplaceholder.typicode.com/todos/8'# update new obj with string version of new_value with updated query parameter
new_value.update({"jsonUrl": new_query})
new_obj = {**original_obj, "value": json.dumps(new_value)}
# add new_obj back to the stream configuration
stream_configuration.append(new_obj)
new_obj{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonSelection',
'type': 'string',
'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}import datetime as dt
stream["description"] = f"updated at {dt.datetime.now()}"
stream["configuration"] = stream_configuration
stream{'id': 1658,
'valid': True,
'invalidExecutionId': None,
'transport': {'type': 'CONNECTOR',
'description': 'com.domo.connector.json.customparsing',
'version': '4'},
'updateMethod': 'REPLACE',
'dataProvider': {'id': 3251,
'key': 'json5',
'name': 'JSON',
'url': None,
'defaultConnectorId': None,
'defaultConnectorLabel': None,
'authenticationScheme': 'fields',
'connectorValidatorPresent': False,
'resourceBundlePresent': False,
'moduleHandler': None,
'iconPicker': False,
'unsafe': False,
'scope': '0',
'producer': 'DOMO',
'version': '0',
'dateCreated': 1601579548000,
'dateUpdated': 1693235941000,
'visibility': 'WITH_CONNECTOR',
'type': 'STANDARD',
'customerOAuthConfigurationDataProviderType': None,
'authenticationSchemeConfiguration': [{'id': 12114,
'name': 'authentication',
'type': 'dropdown',
'text': 'Authentication Type',
'tooltipText': 'Authentication Type',
'required': False,
'encrypted': False,
'defaultValue': 'basic',
'restrictions': None,
'sortOrder': 1,
'data': {'options': [{'label': 'None', 'value': 'none'},
{'label': 'Basic Authentication or SFTP', 'value': 'basic'},
{'label': 'API key', 'value': 'apiKey'}]},
'configField': False},
{'id': 12115,
'name': 'username',
'type': 'string',
'text': 'Username',
'tooltipText': 'Enter a username if using SFTP or Basic Authentication, otherwise leave blank.',
'required': False,
'encrypted': False,
'defaultValue': '',
'restrictions': None,
'sortOrder': 2,
'data': {},
'configField': False},
{'id': 12116,
'name': 'password',
'type': 'password',
'text': 'Password',
'tooltipText': 'Enter a password if using SFTP or Basic Authentication, otherwise leave blank.',
'required': False,
'encrypted': True,
'defaultValue': '',
'restrictions': None,
'sortOrder': 3,
'data': {},
'configField': False},
{'id': 12117,
'name': 'apiType',
'type': 'dropdown',
'text': 'API key type',
'tooltipText': 'API key type',
'required': False,
'encrypted': False,
'defaultValue': 'header',
'restrictions': None,
'sortOrder': 4,
'data': {'options': [{'label': 'Pass as a header', 'value': 'header'},
{'label': 'Pass as a parameter', 'value': 'parameter'}]},
'configField': False},
{'id': 12118,
'name': 'name',
'type': 'string',
'text': 'Parameter/header Name',
'tooltipText': 'Parameter or header name',
'required': False,
'encrypted': False,
'defaultValue': '',
'restrictions': None,
'sortOrder': 5,
'data': {},
'configField': False},
{'id': 12119,
'name': 'apiKey',
'type': 'string',
'text': 'Api Key',
'tooltipText': 'Api Key',
'required': False,
'encrypted': True,
'defaultValue': '',
'restrictions': None,
'sortOrder': 6,
'data': {},
'configField': False}]},
'account': {'id': 91,
'userId': 1893952720,
'name': 'JSON Account',
'displayName': 'JSON Account',
'type': 'data',
'valid': True,
'dataProviderType': 'json5',
'credentialsType': 'fields',
'createdAt': 1707328700000,
'createdBy': 1893952720,
'modifiedAt': 1707328700000,
'modifiedBy': 1893952720,
'configurations': {},
'accountId': 91,
'accountTemplateId': None,
'accountTemplateAuthorizationId': None},
'accountTemplate': None,
'dataSource': {'id': '5f90901a-f0e6-451e-9bc9-09e23e96bd7f',
'displayType': 'json5',
'dataProviderType': 'json5',
'type': 'json5',
'name': 't1',
'description': '',
'owner': {'id': '1893952720',
'name': 'Jae Wilson1',
'type': 'USER',
'group': False},
'status': 'SUCCESS',
'created': 1713471499000,
'lastTouched': 1717526845000,
'lastUpdated': 1717447031461,
'nextUpdate': 1717533419597,
'rowCount': 1,
'columnCount': 6,
'cardInfo': {'cardCount': 0, 'cardViewCount': 0},
'properties': {'formulas': {'formulas': {}}},
'state': 'SUCCESS',
'validConfiguration': True,
'validAccount': True,
'accountId': 91,
'streamId': 1658,
'transportType': 'CONNECTOR',
'adc': False,
'adcExternal': False,
'cloudId': 'domo',
'cloudName': 'Domo',
'permissions': 'READ_WRITE_DELETE_SHARE_ADMIN',
'hidden': False,
'scheduleActive': True,
'cloudEngine': 'domo'},
'schemaDefinition': {'columns': [{'type': 'LONG',
'name': 'id',
'id': 'id',
'visible': True,
'metadata': {'colLabel': 'id', 'colFormat': '', 'isEncrypted': False}},
{'type': 'STRING',
'name': 'completed',
'id': 'completed',
'visible': True,
'metadata': {'colLabel': 'completed',
'colFormat': '',
'isEncrypted': False}},
{'type': 'STRING',
'name': 'title',
'id': 'title',
'visible': True,
'metadata': {'colLabel': 'title', 'colFormat': '', 'isEncrypted': False}},
{'type': 'LONG',
'name': 'userId',
'id': 'userId',
'visible': True,
'metadata': {'colLabel': 'userId', 'colFormat': '', 'isEncrypted': False}},
{'type': 'DOUBLE',
'name': '_BATCH_ID_',
'id': '_BATCH_ID_',
'visible': True,
'metadata': {'colLabel': '_BATCH_ID_',
'colFormat': '',
'isEncrypted': False}},
{'type': 'DATETIME',
'name': '_BATCH_LAST_RUN_',
'id': '_BATCH_LAST_RUN_',
'visible': True,
'metadata': {'colLabel': '_BATCH_LAST_RUN_',
'colFormat': '',
'isEncrypted': False}}]},
'scheduleExpression': '0 37 20 * * ? TZ=UTC',
'scheduleStartDate': None,
'advancedScheduleJson': '{"type":"DAY","at":"08:37 PM","timezone":"UTC"}',
'scheduleRetryExpression': None,
'scheduleRetryCount': 0,
'lastExecution': {'streamId': 1658,
'executionId': 49,
'toe': 'PN218JHC9K-NF07D-UM72F',
'startedAt': 1717447021.0,
'endedAt': 1717447031.0,
'updateMethod': 'REPLACE',
'index': True,
'retryCount': 0,
'retryExecution': None,
'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312',
'uploadId': 49,
'indexRequestKey': 20240603203710.594,
'currentState': 'SUCCESS',
'runType': 'AUTOMATED',
'createdAt': 1717447021.0,
'modifiedAt': 1717447031.0,
'latestPhase': None,
'currentPhase': None,
'removed': False,
'rowsInserted': 1,
'bytesInserted': 63,
'startedBy': None,
'cancelledBy': None,
'dataTag': None,
'peakMemoryUsedBytes': None,
'peakMemoryCommittedBytes': None,
'exportable': False,
'manualIndex': False,
'errors': []},
'lastSuccessfulExecution': {'streamId': 1658,
'executionId': 49,
'toe': 'PN218JHC9K-NF07D-UM72F',
'startedAt': 1717447021.0,
'endedAt': 1717447031.0,
'updateMethod': 'REPLACE',
'index': True,
'retryCount': 0,
'retryExecution': None,
'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312',
'uploadId': 49,
'indexRequestKey': 20240603203710.594,
'currentState': 'SUCCESS',
'runType': 'AUTOMATED',
'createdAt': 1717447021.0,
'modifiedAt': 1717447031.0,
'latestPhase': None,
'currentPhase': None,
'removed': False,
'rowsInserted': 1,
'bytesInserted': 63,
'startedBy': None,
'cancelledBy': None,
'dataTag': None,
'peakMemoryUsedBytes': None,
'peakMemoryCommittedBytes': None,
'exportable': False,
'manualIndex': False,
'errors': []},
'currentExecution': None,
'currentExecutionState': 'SUCCESS',
'createdAt': 1713471499.0,
'createdBy': 1893952720,
'modifiedAt': 1717509420.0,
'modifiedBy': 1893952720,
'scheduleState': 'ACTIVE',
'scheduleAssertion': False,
'inactiveScheduleCode': None,
'configuration': [{'streamId': 1658,
'category': 'METADATA',
'name': '_description_',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'retry.retryNumber',
'type': 'string',
'value': '0'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mergeChoice',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonParsing',
'type': 'string',
'value': '{"parsing":""}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'cloud',
'type': 'string',
'value': 'domo'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'schema',
'type': 'string',
'value': '{"automatedSchema":true}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonPaging',
'type': 'string',
'value': '{"baseUrl":"https://jsonplaceholder.typicode.com/todos/3","tokenType":"path","pagingType":"none","parsing":"","total":"true","pageLimitValue":0,"startPage":"1","encodeParameterValue":true,"encodeParameterKey":true,"pageParameter":"","tokenLocation":"first"}'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.mode',
'type': 'string',
'value': 'REPLACE'},
{'streamId': 1658,
'category': 'METADATA',
'name': 'updatemode.upsertKeys',
'type': 'string',
'value': ''},
{'streamId': 1658,
'category': 'METADATA',
'name': 'jsonSelection',
'type': 'string',
'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}],
'description': 'updated at 2024-06-04 19:47:56.956457'}await stream_routes.update_stream(stream_id=stream_id, auth=auth, body=stream)adjusting num_stacks_to_drop, consider revising `get_traceback` call
{'stack_length': 16, 'module_index': 12, 'num_stacks_to_drop_passed': 3}ResponseGetData(status=200, response={'id': 1658, 'valid': True, 'invalidExecutionId': None, 'transport': {'type': 'CONNECTOR', 'description': 'com.domo.connector.json.customparsing', 'version': '4'}, 'updateMethod': 'REPLACE', 'dataProvider': {'id': 3251, 'key': 'json5', 'name': 'JSON', 'url': None, 'defaultConnectorId': None, 'defaultConnectorLabel': None, 'authenticationScheme': 'fields', 'connectorValidatorPresent': False, 'resourceBundlePresent': False, 'moduleHandler': None, 'iconPicker': False, 'unsafe': False, 'scope': '0', 'producer': 'DOMO', 'version': '0', 'dateCreated': 1601579548000, 'dateUpdated': 1693235941000, 'visibility': 'WITH_CONNECTOR', 'type': 'STANDARD', 'customerOAuthConfigurationDataProviderType': None, 'authenticationSchemeConfiguration': [{'id': 12114, 'name': 'authentication', 'type': 'dropdown', 'text': 'Authentication Type', 'tooltipText': 'Authentication Type', 'required': False, 'encrypted': False, 'defaultValue': 'basic', 'restrictions': None, 'sortOrder': 1, 'data': {'options': [{'label': 'None', 'value': 'none'}, {'label': 'Basic Authentication or SFTP', 'value': 'basic'}, {'label': 'API key', 'value': 'apiKey'}]}, 'configField': False}, {'id': 12115, 'name': 'username', 'type': 'string', 'text': 'Username', 'tooltipText': 'Enter a username if using SFTP or Basic Authentication, otherwise leave blank.', 'required': False, 'encrypted': False, 'defaultValue': '', 'restrictions': None, 'sortOrder': 2, 'data': {}, 'configField': False}, {'id': 12116, 'name': 'password', 'type': 'password', 'text': 'Password', 'tooltipText': 'Enter a password if using SFTP or Basic Authentication, otherwise leave blank.', 'required': False, 'encrypted': True, 'defaultValue': '', 'restrictions': None, 'sortOrder': 3, 'data': {}, 'configField': False}, {'id': 12117, 'name': 'apiType', 'type': 'dropdown', 'text': 'API key type', 'tooltipText': 'API key type', 'required': False, 'encrypted': False, 'defaultValue': 'header', 'restrictions': None, 'sortOrder': 4, 'data': {'options': [{'label': 'Pass as a header', 'value': 'header'}, {'label': 'Pass as a parameter', 'value': 'parameter'}]}, 'configField': False}, {'id': 12118, 'name': 'name', 'type': 'string', 'text': 'Parameter/header Name', 'tooltipText': 'Parameter or header name', 'required': False, 'encrypted': False, 'defaultValue': '', 'restrictions': None, 'sortOrder': 5, 'data': {}, 'configField': False}, {'id': 12119, 'name': 'apiKey', 'type': 'string', 'text': 'Api Key', 'tooltipText': 'Api Key', 'required': False, 'encrypted': True, 'defaultValue': '', 'restrictions': None, 'sortOrder': 6, 'data': {}, 'configField': False}]}, 'account': {'id': 91, 'userId': 1893952720, 'name': 'JSON Account', 'displayName': 'JSON Account', 'type': 'data', 'valid': True, 'dataProviderType': 'json5', 'credentialsType': 'fields', 'createdAt': 1707328700000, 'createdBy': 1893952720, 'modifiedAt': 1707328700000, 'modifiedBy': 1893952720, 'configurations': {}, 'accountId': 91, 'accountTemplateId': None, 'accountTemplateAuthorizationId': None}, 'accountTemplate': None, 'dataSource': {'id': '5f90901a-f0e6-451e-9bc9-09e23e96bd7f', 'displayType': 'json5', 'dataProviderType': 'json5', 'type': 'json5', 'name': 't1', 'description': '', 'owner': {'id': '1893952720', 'name': 'Jae Wilson1', 'type': 'USER', 'group': False}, 'status': 'SUCCESS', 'created': 1713471499000, 'lastTouched': 1717530477000, 'lastUpdated': 1717447031461, 'nextUpdate': 1717533419546, 'rowCount': 1, 'columnCount': 6, 'cardInfo': {'cardCount': 0, 'cardViewCount': 0}, 'properties': {'formulas': {'formulas': {}}}, 'state': 'SUCCESS', 'validConfiguration': True, 'validAccount': True, 'accountId': 91, 'streamId': 1658, 'transportType': 'CONNECTOR', 'adc': False, 'adcExternal': False, 'cloudId': 'domo', 'cloudName': 'Domo', 'permissions': 'READ_WRITE_DELETE_SHARE_ADMIN', 'hidden': False, 'scheduleActive': True, 'cloudEngine': 'domo'}, 'schemaDefinition': {'columns': [{'type': 'LONG', 'name': 'id', 'id': 'id', 'visible': True, 'metadata': {'colLabel': 'id', 'colFormat': '', 'isEncrypted': False}}, {'type': 'STRING', 'name': 'completed', 'id': 'completed', 'visible': True, 'metadata': {'colLabel': 'completed', 'colFormat': '', 'isEncrypted': False}}, {'type': 'STRING', 'name': 'title', 'id': 'title', 'visible': True, 'metadata': {'colLabel': 'title', 'colFormat': '', 'isEncrypted': False}}, {'type': 'LONG', 'name': 'userId', 'id': 'userId', 'visible': True, 'metadata': {'colLabel': 'userId', 'colFormat': '', 'isEncrypted': False}}, {'type': 'DOUBLE', 'name': '_BATCH_ID_', 'id': '_BATCH_ID_', 'visible': True, 'metadata': {'colLabel': '_BATCH_ID_', 'colFormat': '', 'isEncrypted': False}}, {'type': 'DATETIME', 'name': '_BATCH_LAST_RUN_', 'id': '_BATCH_LAST_RUN_', 'visible': True, 'metadata': {'colLabel': '_BATCH_LAST_RUN_', 'colFormat': '', 'isEncrypted': False}}]}, 'scheduleExpression': '0 37 20 * * ? TZ=UTC', 'scheduleStartDate': None, 'advancedScheduleJson': '{"type":"DAY","at":"08:37 PM","timezone":"UTC"}', 'scheduleRetryExpression': None, 'scheduleRetryCount': 0, 'lastExecution': {'streamId': 1658, 'executionId': 49, 'toe': 'PN218JHC9K-NF07D-UM72F', 'startedAt': 1717447021.0, 'endedAt': 1717447031.0, 'updateMethod': 'REPLACE', 'index': True, 'retryCount': 0, 'retryExecution': None, 'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312', 'uploadId': 49, 'indexRequestKey': 20240603203710.594, 'currentState': 'SUCCESS', 'runType': 'AUTOMATED', 'createdAt': 1717447021.0, 'modifiedAt': 1717447031.0, 'latestPhase': None, 'currentPhase': None, 'removed': False, 'rowsInserted': 1, 'bytesInserted': 63, 'startedBy': None, 'cancelledBy': None, 'dataTag': None, 'peakMemoryUsedBytes': None, 'peakMemoryCommittedBytes': None, 'exportable': False, 'manualIndex': False, 'errors': []}, 'lastSuccessfulExecution': {'streamId': 1658, 'executionId': 49, 'toe': 'PN218JHC9K-NF07D-UM72F', 'startedAt': 1717447021.0, 'endedAt': 1717447031.0, 'updateMethod': 'REPLACE', 'index': True, 'retryCount': 0, 'retryExecution': None, 'containerManagerId': 'cm-c1240a27-2d9b-49ab-835e-8d11c8f56312', 'uploadId': 49, 'indexRequestKey': 20240603203710.594, 'currentState': 'SUCCESS', 'runType': 'AUTOMATED', 'createdAt': 1717447021.0, 'modifiedAt': 1717447031.0, 'latestPhase': None, 'currentPhase': None, 'removed': False, 'rowsInserted': 1, 'bytesInserted': 63, 'startedBy': None, 'cancelledBy': None, 'dataTag': None, 'peakMemoryUsedBytes': None, 'peakMemoryCommittedBytes': None, 'exportable': False, 'manualIndex': False, 'errors': []}, 'currentExecution': None, 'currentExecutionState': 'SUCCESS', 'createdAt': 1713471499.0, 'createdBy': 1893952720, 'modifiedAt': 1717509420.0, 'modifiedBy': 1893952720, 'scheduleState': 'ACTIVE', 'scheduleAssertion': False, 'inactiveScheduleCode': None, 'configuration': [{'streamId': 1658, 'category': 'METADATA', 'name': 'updatemode.upsertKeys', 'type': 'string', 'value': ''}, {'streamId': 1658, 'category': 'METADATA', 'name': 'updatemode.mergeChoice', 'type': 'string', 'value': ''}, {'streamId': 1658, 'category': 'METADATA', 'name': 'jsonParsing', 'type': 'string', 'value': '{"parsing":""}'}, {'streamId': 1658, 'category': 'METADATA', 'name': 'jsonPaging', 'type': 'string', 'value': '{"baseUrl":"https://jsonplaceholder.typicode.com/todos/3","tokenType":"path","pagingType":"none","parsing":"","total":"true","pageLimitValue":0,"startPage":"1","encodeParameterValue":true,"encodeParameterKey":true,"pageParameter":"","tokenLocation":"first"}'}, {'streamId': 1658, 'category': 'METADATA', 'name': 'jsonSelection', 'type': 'string', 'value': '{"connectionMethod": "Advanced", "jsonUrl": "https://jsonplaceholder.typicode.com/todos/8", "httpMethod": "GET", "jsonLineReader": "false", "escapeBackslash": "false", "httpsHeaders": [], "body": "", "queryParameters": [], "useDateFilter": false, "dateSelection": {"dateType": "date_range", "startDate": {"type": "relative", "offset": 0}, "endDate": {"type": "relative", "offset": 0}}, "dateQueryParamsFormat": "yyyy-MM-dd", "certificateInputType": "NoCertificate", "encodeParameterKey": true, "encodeParameterValue": true, "datasetType": "static", "dateParameter": {"dateType": "single_date", "dateFrom": "relative", "dateFromOffset": 1, "dateTo": "relative", "dateToOffset": 0, "date": "relative", "dateOffset": 1}, "useBody": false, "dynamicValuesType": "enter", "rateLimitSelection": "false", "timeUnit": "second", "dateParameterType": "separate"}'}, {'streamId': 1658, 'category': 'METADATA', 'name': '_description_', 'type': 'string', 'value': ''}, {'streamId': 1658, 'category': 'METADATA', 'name': 'cloud', 'type': 'string', 'value': 'domo'}, {'streamId': 1658, 'category': 'METADATA', 'name': 'updatemode.mode', 'type': 'string', 'value': 'REPLACE'}, {'streamId': 1658, 'category': 'METADATA', 'name': 'schema', 'type': 'string', 'value': '{"automatedSchema":true}'}, {'streamId': 1658, 'category': 'METADATA', 'name': 'retry.retryNumber', 'type': 'string', 'value': '0'}]}, is_success=True, parent_class=None)