{"type":"doc","content":[{"type":"bodiedExtension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"scroll-ignore","parameters":{"macroParams":{"scroll-viewport":{"value":"true"},"scroll-pdf":{"value":"true"},"scroll-office":{"value":"true"},"scroll-chm":{"value":"true"},"scroll-docbook":{"value":"true"},"scroll-eclipsehelp":{"value":"true"},"scroll-html":{"value":"true"},"scroll-epub":{"value":"true"}},"macroMetadata":{"macroId":{"value":"5756812f-98f6-4fd4-bc9f-2ae911a13a05"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://scroll-exporter-extensions.us.exporter.k15t.app/static/assets/images/macro-icons/scroll-ignore.svg"}}],"title":"Scroll Ignore"}},"localId":"23136790-f63e-4828-a7ad-c5a0677cbd03"},"content":[{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Open in new tab","type":"text","marks":[{"type":"link","attrs":{"href":"https://kada-knowledge-library.scrollhelp.site/home/sql-server-via-collector-method-v3-0-1"}}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"About Collectors","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"excerpt-include","parameters":{"macroParams":{"":{"value":"Collector Method"},"name":{"value":"about"}},"macroMetadata":{"macroId":{"value":"ea8bf8b8-9759-44bc-a446-42aa5ba2e22a"},"schemaVersion":{"value":"1"},"title":"Insert excerpt"}},"localId":"1d974858-b7fc-497a-973b-3637f8202d06"}},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Pre-requisites","type":"text"}]},{"type":"paragraph","content":[{"text":"Collector server minimum requirements","type":"text","marks":[{"type":"strong"}]},{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.macro.core","extensionKey":"excerpt-include","parameters":{"macroParams":{"":{"value":"Collector Method"},"name":{"value":"CollectorServerSpec"},"nopanel":{"value":"true"}},"macroMetadata":{"macroId":{"value":"0d5b5015-f8eb-4444-b9a1-5bc1a5426ce8"},"schemaVersion":{"value":"1"},"title":"Insert excerpt"}},"localId":"1df83dd8-7021-4b7d-8800-b20b401642c6"}}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Databricks Requirements","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"719733b7-a1b8-4383-a65d-a0807138ac4b"}},{"type":"strong"}]},{"type":"hardBreak"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Unity enabled catalogue. ","type":"text"},{"text":"Hive catalogues are not supported currently.","type":"text","marks":[{"type":"strong"},{"type":"em"}]}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"https://community.databricks.com/t5/bangalore/how-do-we-enable-unity-catalog-for-our-workspace/td-p/73258","type":"text","marks":[{"type":"link","attrs":{"href":"https://community.databricks.com/t5/bangalore/how-do-we-enable-unity-catalog-for-our-workspace/td-p/73258"}}]}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Enable System Schemas for","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"access","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"query","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Follow the following documentation to enable ","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.databricks.com/en/admin/system-tables/index.html#enable"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://docs.databricks.com/en/dev-tools/auth/pat.html"}}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://kb.databricks.com/unity-catalog/find-your-metastore-id"}}]},{"type":"codeBlock","content":[{"text":"curl -v -X PUT -H \"Authorization: Bearer \" \"https://.cloud.databricks.com/api/2.0/unity-catalog/metastores//systemschemas/access\"\ncurl -v -X PUT -H \"Authorization: Bearer \" \"https://.cloud.databricks.com/api/2.0/unity-catalog/metastores//systemschemas/query\"","type":"text"}]}]}]}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 1: Create the Source in K","type":"text"}]},{"type":"paragraph","content":[{"text":"Create a source in K","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Go to ","type":"text"},{"text":"Settings","type":"text","marks":[{"type":"strong"}]},{"text":", Select ","type":"text"},{"text":"Sources","type":"text","marks":[{"type":"strong"}]},{"text":" and click ","type":"text"},{"text":"Add Source","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Select ","type":"text"},{"text":"“Load from File” option","type":"text","marks":[{"type":"strong"}]},{"type":"hardBreak"}]}]},{"type":"listItem","content":[{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":862,"id":"7683c2ec-8ef1-439c-a756-1d8146126773","collection":"contentId-2808545301","type":"file","height":1087}}]},{"type":"paragraph","content":[{"text":"Give the source a ","type":"text"},{"text":"Name","type":"text","marks":[{"type":"strong"}]},{"text":" - e.g. Databricks Production ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Add the ","type":"text"},{"text":"Host name","type":"text","marks":[{"type":"strong"}]},{"text":" for the Databricks Instance","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Click ","type":"text"},{"text":"Next ","type":"text","marks":[{"type":"strong"}]},{"text":"& ","type":"text"},{"text":"Finish Setup","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 2: Getting Access to the Source Landing Directory","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"excerpt-include","parameters":{"macroParams":{"":{"value":"Collector Method"},"name":{"value":"landing"}},"macroMetadata":{"macroId":{"value":"3dee3511-0c8c-4afc-a749-699e80e86de0"},"schemaVersion":{"value":"1"},"title":"Insert excerpt"}},"localId":"df9aa353-b7ec-4190-a724-c3ca6717ef5d"}},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 3: Install the Collector","type":"text"}]},{"type":"paragraph","content":[{"text":"It is recommended to use a python environment such as ","type":"text"},{"text":"pyenv","type":"text","marks":[{"type":"strong"}]},{"text":" or ","type":"text"},{"text":"pipenv","type":"text","marks":[{"type":"strong"}]},{"text":" if you are not intending to install this package at the system level.","type":"text"}]},{"type":"paragraph","content":[{"text":"Some python packages also have dependencies on the OS level packages, so you may be required to install additional OS packages if the below fails to install.","type":"text"}]},{"type":"paragraph","content":[{"text":"You can download the Latest Core Library and Databricks whl via ","type":"text"},{"text":"Platform Settings → Sources","type":"text","marks":[{"type":"strong"}]},{"text":" → ","type":"text"},{"text":"Download Collectors","type":"text","marks":[{"type":"strong"}]}]},{"type":"mediaSingle","attrs":{"layout":"center"},"content":[{"type":"media","attrs":{"width":899,"id":"32ef8f55-437e-47b0-9c51-0001fd7cf1e4","collection":"contentId-2808545301","type":"file","height":439}}]},{"type":"paragraph","content":[{"text":"Run the following command to install the collector","type":"text"}]},{"type":"codeBlock","content":[{"text":"pip install kada_collectors_extractors_databricks-3.0.0-py3-none-any.whl","type":"text"}]},{"type":"paragraph","content":[{"text":"You will also need to install the corresponding common library kada_collectors_lib-x.x.x for this collector to function properly.","type":"text"}]},{"type":"codeBlock","content":[{"text":"pip install kada_collectors_lib-x.x.x-py3-none-any.whl","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 4: Configure the Collector","type":"text"}]},{"type":"paragraph","content":[{"text":"The collector requires a set of parameters to connect to and extract metadata from Databricks.","type":"text"}]},{"type":"table","attrs":{"layout":"default","width":760.0,"localId":"92b4ef3f-a0cc-42f2-a08c-7efd9d781efd"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"FIELD","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"FIELD TYPE","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"DESCRIPTION","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"EXAMPLE","type":"text","marks":[{"type":"strong"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"access_token","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Databricks personal access token for authentication","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"server_hostname","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Server address to the Databricks Service","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":"adb-..azuredatabricks.net","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"http_path","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"var(--ds-surface, #FFFFFF)","rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Http path either to a DBSQL endpoint or to a DBR interactive cluster","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"/sql/1.0/warehouses/","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"statement_timeout","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"integer","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Query time limit (in seconds). Query execution will be timed out after this duration, if set. Default is 600s.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"600","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"host","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"The onboarded host value in K, generally this would be the same as the server value, depending on what you onboard it as.","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph"}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"databases","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"list","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"list of databases to extract (catalogs in Databricks)","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"[“dwh”, “adw”]","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"information_catalog","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"The catalog to extract information. by default is 'system' which hold all the metadata for all the databases","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"system","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"output_path","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"string","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Absolute path to the output location where files are to be written","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"“/tmp/output”","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"mask","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"boolean","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"To enable masking or not","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"true","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"compress","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"boolean","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"To gzip the output or not","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"true","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"meta_only","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"boolean","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"Do you want to extract metadata only without enabling extended events?","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":[170.0]},"content":[{"type":"paragraph","content":[{"text":"false","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"These parameters can be added directly into the run or you can use pass the parameters in via a JSON file. The following is an example you can use that is included in the example run code below.","type":"text"}]},{"type":"paragraph","content":[{"text":"kada_databricks_extractor_config.json ","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","attrs":{"language":"json"},"content":[{"text":"{\n \"access_token\": \"\",\n \"server_hostname\": \"\",\n \"http_path\": \"\",\n \"statement_timeout\": 600,\n \"host\": \"\",\n \"databases\": [],\n \"information_catalog\": \"system\",\n \"output_path\": \"/tmp/output\",\n \"mask\": true,\n \"compress\": true,\n \"meta_only\": true\n}","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 5: Run the Collector","type":"text"}]},{"type":"paragraph","content":[{"text":"The following code is an example of how to run the extractor. You may need to uplift this code to meet any code standards at your organisation. ","type":"text"}]},{"type":"paragraph","content":[{"text":"This can be executed in any python environment where the whl has been installed. ","type":"text"}]},{"type":"paragraph","content":[{"text":"This code sample uses the ","type":"text"},{"text":"kada_databricks_extractor.py","type":"text","marks":[{"type":"strong"}]},{"text":" for handling the configuration details","type":"text"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"text":"import os\nimport argparse\nfrom kada_collectors.extractors.utils import load_config, get_hwm, publish_hwm, get_generic_logger\nfrom kada_collectors.extractors.databricks import Extractor\n\nget_generic_logger('root') # Set to use the root logger, you can change the context accordingly or define your own logger\n\n_type = 'databricks'\ndirname = os.path.dirname(__file__)\nfilename = os.path.join(dirname, 'kada_{}_extractor_config.json'.format(_type))\n\nparser = argparse.ArgumentParser(description='KADA Databricks Extractor.')\nparser.add_argument('--config', '-c', dest='config', default=filename, help='Location of the configuration json, default is the config json in the same directory as the script.')\nparser.add_argument('--name', '-n', dest='name', default=_type, help='Name of the collector instance.')\nargs = parser.parse_args()\n\nstart_hwm, end_hwm = get_hwm(args.name)\n\n# Initialize and run the extractor\next = Extractor(**load_config(args.config))\next.test_connection()\next.run(**{\"start_hwm\": start_hwm, \"end_hwm\": end_hwm})\n\n# Publish high watermark\npublish_hwm(args.name, end_hwm)","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Advance options:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"If you wish to maintain your own high water mark files elsewhere you can use the above section’s script as a guide on how to call the extractor. The configuration file is simply the keyword arguments in JSON format. Refer to this document for more information ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://kadaai.atlassian.net/wiki/spaces/KSL/pages/1902411777/Additional+Notes#Storing-HWM-in-another-location"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"If you are handling external arguments of the runner yourself, you’ll need to consider additional items for the run method. Refer to this document for more information ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://kadaai.atlassian.net/wiki/spaces/KSL/pages/1902411777/Additional+Notes#The-run-method"}},{"text":" ","type":"text"}]},{"type":"rule"},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"text":"class Extractor(access_token: str, server_hostname: str, http_path: str, \\\n statement_timeout: int = 600, host: str = None, databases: list = [], \\\n information_catalog: str = 'system', output_path: str = './output', \\\n mask: bool = False, compress: bool = False, meta_only: bool = False) -> None","type":"text"}]},{"type":"paragraph","content":[{"text":"access_token: Databricks personal access token authentication","type":"text"},{"type":"hardBreak"},{"text":"server_hostname: Server address to the Databricks Service","type":"text"},{"type":"hardBreak"},{"text":"http_path: Http path either to a DBSQL endpoint or to a DBR interactive cluster","type":"text"},{"type":"hardBreak"},{"text":"statement_timeout: Query time limit (in seconds). Query execution will be timed out after this duration, if set. Default is 600s.","type":"text"},{"type":"hardBreak"},{"text":"host: the onboarded host value in K, generally it will be the same as the server","type":"text"},{"type":"hardBreak"},{"text":"databases: list of databases to extract (catalogs in Databricks)","type":"text"},{"type":"hardBreak"},{"text":"information_catalog: The catalog to extract information. by default is 'system' which hold all the metadata for all the databases","type":"text"},{"type":"hardBreak"},{"text":"output_path: full or relative path to where the outputs should go","type":"text"},{"type":"hardBreak"},{"text":"mask: To mask the META/DATABASE_LOG files or not","type":"text"},{"type":"hardBreak"},{"text":"compress: To gzip output files or not","type":"text"},{"type":"hardBreak"},{"text":"meta_only: To extract without extended events or not","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 6: Check the Collector Outputs ","type":"text"}]},{"type":"paragraph","content":[{"text":"K Extracts ","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"A set of files (eg metadata, databaselog, linkages, events etc) will be generated. These files will appear in the output_path directory you set in the configuration details","type":"text"}]},{"type":"paragraph","content":[{"text":"High Water Mark File","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"A high water mark file is created in the same directory as the execution called ","type":"text"},{"text":"databricks_hwm.txt","type":"text","marks":[{"type":"strong"}]},{"text":" and produce files according to the configuration JSON. This file is only produced if you call the publish_hwm method. ","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Step 7: Push the Extracts to K","type":"text"}]},{"type":"paragraph","content":[{"text":"Once the files have been validated, you can push the files to the ","type":"text"},{"text":"K landing directory","type":"text","marks":[{"type":"link","attrs":{"href":"#"}}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"You can use ","type":"text"},{"text":"Azure Storage Explorer","type":"text","marks":[{"type":"link","attrs":{"href":"https://azure.microsoft.com/en-us/features/storage-explorer/"}}]},{"text":" if you want to initially do this manually. You can push the files using python as well (see Airflow example below)","type":"text"}]},{"type":"rule"},{"type":"heading","attrs":{"level":2},"content":[{"text":"Example: Using Airflow to orchestrate the Extract and Push to K","type":"text"}]},{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"excerpt-include","parameters":{"macroParams":{"":{"value":"Collector Method"},"name":{"value":"airflow"}},"macroMetadata":{"macroId":{"value":"55285bd9-b927-4df8-9c43-39adfe815b13"},"schemaVersion":{"value":"1"},"title":"Insert excerpt"}},"localId":"c09c439a-0b08-4611-a707-ee5e7bebc0c5"}},{"type":"paragraph"},{"type":"paragraph"}],"version":1}