Informatica (via Collector method) - v3.0.0

pip install kada_collectors_extractors_<version>-none-any.whl

pip install kada_collectors_lib-<version>-none-any.whl

select 
'call infacmd.bat isp getsessionlog -dn <INFORMATICA_DOMAIN> -hp <HOST>:<PORT> -un <SERVER USERNAME> -pd <SERVER PASSWORD> -is <SERVERNAME> -rs <REPO NAME> -ru <REPO USERNAME> -rp <REPO PASSWORD> -fm xml -fn ' || ws.subject_area || ' -wf ' || ws.workflow_name || ' -ss ' || CASE WHEN hierarchy_structure is null then ws.instance_name ELSE '"' || substr(hierarchy_structure, 2) || '"' END || ' -lo <C:\\output\\path\\for\\logs\\>' || ws.workflow_id || '_' || ws.task_id || '_' || ws.instance_id as cmd
from (
    SELECT ti.instance_name,
        ti.task_id,
        ti.version_number,
        wws.instance_id,
        wf.workflow_id,
        wf.workflow_name,
        wf.workflow_comments,
        wf.server_name,
        wf.subject_area,
        hierarchy_structure,
        path
FROM (
        select path 
        , TO_NUMBER(substr(path, 2, instr(path,'/',1, 2)-2)) as workflow_id
        , TO_NUMBER(substr(path, -instr(reverse(path),'/', 1, 2)+1, instr(reverse(path),'/', 1, 2)-2)) as task_id
        , hierarchy_structure
        , instance_id
        from (SELECT DISTINCT '/' || temp1.task_id AS path
                , temp1.task_name AS hierarchy_structure
                , 0 as instance_id
                FROM opb_task temp1, opb_subject temp2
                WHERE temp1.subject_id = temp2.subj_id
                AND temp1.task_type = 71 -- workflows
                UNION ALL
                SELECT DISTINCT temp1.path
                    , temp1.task_name AS hierarchy_structure
                    , instance_id
                FROM (SELECT opb_task_inst.workflow_id, opb_task_inst.task_id, opb_task_inst.instance_id, LEVEL depth,
                        SYS_CONNECT_BY_PATH(opb_task_inst.workflow_id ,'/') || '/' || opb_task_inst.task_id || '/' path,
                        SYS_CONNECT_BY_PATH(opb_task_inst.instance_name ,'/') task_name
                        FROM opb_task_inst
                        WHERE opb_task_inst.task_type IN (68,70)
                        START WITH workflow_id IN (select distinct w.workflow_id
                                                        from rep_workflows w
                                                        join rep_task_inst ti on w.workflow_id = ti.workflow_id
                                                        where ti.task_type_name = 'Worklet'
                                                        and w.subject_area not in ('<SUBJECT_AREAS_TO_EXCLUDE>')
                                                    )
                        CONNECT BY PRIOR opb_task_inst.task_id = opb_task_inst.workflow_id
                    ) temp1,
                    opb_task temp2,
                    opb_subject temp3
                WHERE temp2.subject_id = temp3.subj_id
                AND temp2.task_id = SUBSTR(temp1.path,2, INSTR(temp1.path,'/', 1, 2) -2 )
                ORDER BY path ASC )
        where instance_id <> 0
) wws
    JOIN rep_task_inst ti on ti.task_id = wws.task_id and ti.task_type = 68
    JOIN REP_WORKFLOWS wf on wws.workflow_id = wf.workflow_id
UNION
SELECT ti.instance_name,
    ti.task_id,
    ti.version_number,
    ti.instance_id,
    wf.workflow_id,
    wf.workflow_name,
    wf.workflow_comments,
    wf.server_name,
    wf.subject_area,
    '' as hierarchy_structure,
    '' as path
FROM REP_WORKFLOWS wf
    JOIN rep_task_inst ti on ti.workflow_id = wf.workflow_id and ti.task_type = 68
where wf.subject_area not in ('<SUBJECT_AREAS_TO_EXCLUDE>')
) ws
    join (select distinct workflow_id as workflow_id from rep_wflow_run) active_wflows on ws.workflow_id = active_wflows.workflow_id 

@echo off
cd /d C:
cd "C:\Informatica\9.1.0\clients\DeveloperClient\infacmd"
echo %cd%
<ADD CALLS from SQL here>
pause

import os
import argparse
from kada_collectors.extractors.utils import load_config, get_generic_logger
from kada_collectors.extractors.informatica import runtime_parser

get_generic_logger('root') # Set to use the root logger, you can change the context accordingly or define your own logger

_type = 'informatica_runtime_parser'
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, 'kada_{}_extractor_config.json'.format(_type))

parser = argparse.ArgumentParser(description='KADA Informatica Runtime Parser.')
parser.add_argument('--config', '-c', dest='config', default=filename, help='Location of the configuration json, default is the config json in the same directory as the script.')
args = parser.parse_args()

config_args = load_config(args.config)

runtime_parser(**{"input_path": config_args["input_path"], "output_path": config_args["output_path"]})

{
    "username": "",
    "password": "",
    "dsn": "",
    "repo_owner": "",
    "oracle_client_path": "",
    "cached": false,
    "input_path": "/tmp/input",
    "output_path": "/tmp/output",
    "mask": true,
    "compress" true
}

import os
import argparse
from kada_collectors.extractors.utils import load_config, get_hwm, publish_hwm, get_generic_logger
from kada_collectors.extractors.informatica import Extractor

get_generic_logger('root') # Set to use the root logger, you can change the context accordingly or define your own logger

_type = 'informatica'
dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, 'kada_{}_extractor_config.json'.format(_type))

parser = argparse.ArgumentParser(description='KADA Informatica Extractor.')
parser.add_argument('--config', '-c', dest='config', default=filename, help='Location of the configuration json, default is the config json in the same directory as the script.')
args = parser.parse_args()

start_hwm, end_hwm = get_hwm(_type)

ext = Extractor(**load_config(args.config))
ext.test_connection()
ext.run(**{"start_hwm": start_hwm, "end_hwm": end_hwm})

publish_hwm(_type, end_hwm)

from kada_collectors.extractors.snowflake import Extractor

kwargs = {my args} # However you choose to construct your args
hwm_kwrgs = {"start_hwm": "end_hwm": } # The hwm values

ext = Extractor(**kwargs)
ext.run(**hwm_kwrgs)

class Extractor(username: str = None, password: str = None, dsn: str = None, \
    repo_owner: str = None, oracle_client_path: str = None, \
    cached: bool = False, input_path: str = './input', \
    output_path: str = './output', mask: bool = False, compress: bool = False) -> None

from kada_collectors.extractors.informatica import runtime_parser

kwargs = {my args} # However you choose to construct your args

runtime_parser(**kwargs)

def runtime_parser: (input_path: str = './input', output_path: str = './output') -> None