CREATE OR REPLACE PROCEDURE proc_init_job(character varying(50)) RETURNS boolean LANGUAGE NZPLSQL AS BEGIN_PROC DECLARE pjobGroup ALIAS FOR $1;

-pJobID ALIAS FOR $2; vJobId int; vRunStatus varchar(20); vErrorMessage varchar(255); vStep varchar(10); BEGIN vJobId := -1; vRunStatus := ''; vStep := '1.0'; vJobId := (SELECT nvl(MAX(job_sid),0) FROM META_CTL_JOB WHERE JOB_GROUP=pjobGroup); if vJobId > 0 then vRunStatus := (SELECT nvl(JOB_STATUS,'') FROM META_CTL_JOB where job_sid=vJobId); if (vRunStatus = 'RUNNING') then vErrorMessage := (SELECT 'Another Instance of Job with I D :'|| cast(vJobId as varchar(15))|| ' is with status:'||vRunStatus); vErrorMessage := (SELECT 'Error Occured at Step:'|| vSte p ||':'|| vErrorMessage); --select vErrorMessage; vJobId := -1; else vJobId := (SELECT next value for job_sid); INSERT INTO META_CTL_JOB(job_sid,JOB_STATUS,STAR T_DT,EXECUTED_BY,JOB_GROUP) VALUES(vJobID, 'RUNNING',now(),current_user,pjob Group); end if; else vJobId := (SELECT next value for job_sid); INSERT INTO META_CTL_JOB(job_sid, JOB_STATUS,START_DT,EXECUTED_B Y,JOB_GROUP) VALUES(vJobID,'RUNNING',now(),current_user,pjobGroup); end if; END; END_PROC;

=============================================================================== CREATE OR REPLACE PROCEDURE proc_init_process_exec(integer, character varying(10 0), character varying(50), character varying(50), integer) RETURNS integer LANGUAGE NZPLSQL AS BEGIN_PROC DECLARE pJobID ALIAS FOR $1; pProcessName alias for $2; pProcessType alias for $3; pProcessSubType alias for $4; pProcessRunID alias for $5; vRunStatus varchar(20); vpriorProcFailedInd integer; vErrorMessage varchar(255); vStep varchar(10); vProcessID integer; vCurProcGroup varchar(50); vProcessRunId integer; vCount integer; BEGIN vRunStatus := ''; vpriorProcFailedInd := 0; vStep := '1.0'; vProcessID := 0; vCurProcGroup := ''; --vProcessRunId := (select pProcessRunId); /******************************************************* * in order to failed proc if there is no record for 'wkf_Integrate _mb_campaign_member' *********************************************************/ if pProcessName='wkf_Integrate_mb_campaign_member' then vCount := (select count(*) from dell_lm_target..vw_pending_inqui ries); if vCount = 0 then BEGIN vErrorMessage := 'No record return for wkf_Integ rate_mb_campaign_member'; RAISE EXCEPTION 'Error: %', vErrorMessage; return -1; END; end if; end if; if pJobId <= 0 then vErrorMessage := 'No Job is Running State Or Not Set in Param File Properly'; vErrorMessage := (select 'Error Occured at Step: '|| vStep || ' :'|| vErrorMessage); RAISE notice 'Error: %', vErrorMessage; return -1;

end if; vStep := '1.1';

--Getting Process ID vProcessID := (select nvl(MAX(process_sid),0) FROM META_CTL_PROCESS where PROCESS_QUALIFIER=pProcessName and PROCESS_TYPE=pProcessType); vCurProcGroup := (select nvl(MAX(PROCESS_GROUP),'') FROM META_CTL_PROCESS where PROCESS_QUALIFIER=pProcessName and PROCESS_TYPE=pProcessType); if vProcessID <=0 then vErrorMessage := (select 'Process with name: ' || pProcessName | | ' with Process Type of ' || pProcessType || ' not exist in MTA_CTL_PROCESS tab le'); vErrorMessage := (select 'Error Occured at Step:' || vStep || ' :' || vErrorMessage); RAISE notice 'Error: %', vErrorMessage; return -1; end if; vStep := '1.2';

vpriorProcFailedInd := ( select MAX(CASE WHEN STATUS='FAILED' THEN 1 ELSE 0 END) from META_CTL_PROCESS_STATUS PR JOIN ( SELECT PS.job_sid, PS.process_sid, MAX(process_run_sid) as process_ run_sid FROM META_CTL_PROCESS_STATUS PS join META_CTL_JOB JB ON PS.job_sid=JB.job_sid AND JB.job_sid=pJobId join META_CTL_PROCESS P ON PS.process_sid=P.process_sid WHERE nvl(P.PROCESS_GROUP,'')= vCurProcGroup and PS.process_sid != vProcessID GROUP BY PS.job_sid,PS.process_sid ) QRY ON PR.process_run_sid=QRY.process_run_sid);

if vpriorProcFailedInd = 1 then vErrorMessage := 'Prior Processes Failed .. Please Fix the Issue And re-start';

vErrorMessage := (select 'Error Occured at Step:'|| vStep ||':'| | vErrorMessage); RAISE notice 'Error: %', vErrorMessage; return -1; end if; -- UPDATE JOB IF IT'S STATUS IS FAILED UPDATE META_CTL_JOB SET JOB_STATUS='RUNNING' WHERE job_sid=pJobId; vStep := '1.3'; -- Getting Process Run ID vProcessRunID := (select MAX(process_run_sid) FROM META_CTL_PROCESS_STAT US WHERE job_sid=pJobId AND process_sid=vProcessId); if vProcessRunID > 0 then vStep := '1.4'; vRunStatus := (select STATUS FROM META_CTL_PROCESS_STATUS where process_run_sid = vProcessRunID); if (vRunStatus = 'RUNNING' OR vRunStatus = 'SUCCEEDED') then vErrorMessage := 'Another Instance of Process with ID :' || cast(vProcessRunID as varchar(15))|| ' is with status:'|| vRunStatus; vErrorMessage := 'Error Occured at Step:'|| vStep ||':'| | vErrorMessage; RAISE notice 'Error: %', vErrorMessage; vProcessRunID := -1; return vProcessRunID; else vProcessRunID := (select next value for process_run_sid) ; INSERT INTO META_CTL_PROCESS_STATUS(process_run_sid,proc ess_sid,job_sid,START_DT,EXECUTED_BY,STATUS) VALUES(vProcessRunID,vProcessID,pJobId,now(),current_use r,'RUNNING');

end if; else vProcessRunID := (select next value for process_run_sid); INSERT INTO META_CTL_PROCESS_STATUS(process_run_sid, process_sid ,job_sid,START_DT,EXECUTED_BY,STATUS) VALUES(vProcessRunID,vProcessID,pJobId,now(),current_user,'RUNNI NG');

end if; if pProcessRunID > 0 and pProcessSubType like '%TARGET' then INSERT INTO META_PROCESS_FEED_STATUS( process_run_sid, process_sid, file_sid, STATUS, START_DATE) SELECT vProcessRunID, nvl(vProcessID,0), file_sid, 'RUNNING', now() FROM vw_get_pending_files where PROCESS_QUALIFIER=pProcessName; end if; return vProcessRunID; END; END_PROC; ================================================================================ === CREATE OR REPLACE PROCEDURE proc_set_process_status(integer, integer, character varying(100), character varying(50), character varying(20), character varying(40 00)) RETURNS integer LANGUAGE NZPLSQL AS BEGIN_PROC DECLARE pJobID ALIAS FOR $1; pProcessRunID ALIAS FOR $2; pProcessName ALIAS FOR $3; pProcessType ALIAS FOR $4; pProcessStatus ALIAS FOR $5; pProcessMessage ALIAS FOR $6; vRunStatus varchar(20); vErrorMessage varchar(4000); ErrorType varchar(50); Step varchar(10); vFileID int; tableHTML VARCHAR(8000); Proc_Name VARCHAR(150); errorFlag int; subMessage varchar(4000); toMailUsers varchar(4000); vProcessID int; vProcessName varchar(255); vTgtTableName varchar(255); mailFile varchar(255); ourinsertCount int; ourupdateCount int; sql varchar(8000);

nsql varchar(8000); sql_part varchar(8000); tb_name varchar(255); mountPoint varchar(100); exportDir varchar(255); exportfile varchar(250); result integer; pdmr VARCHAR(8000); Proc_Name2 VARCHAR(150); cmd nvarchar(1000); vProcessRunId integer; BEGIN

tb_name := (select 'dell_smb_stage.tmp_'||vTgtTableName); mountPoint :=''; exportDir := ''; exportFile :=''; vRunStatus := ''; vErrorMessage := ''; Step := '1.0'; vFileID := 0; errorFlag := 0; errorType := ''; subMessage := ''; toMailUsers := ''; vProcessID := 0; vProcessName := ''; vTgtTableName := ''; mailFile := ''; ourinsertCount := 0; ourupdateCount := 0; vProcessRunId := (select pProcessRunId); -- if no process run id passed as parameter , get it from most recent pr ocess Step := '1.1'; -print Step+': Getting Latest Process Run Id' if vProcessRunID=0 then vProcessID := (select nvl(MAX(process_sid),0) FROM META_CTL_PROCESS where PROCESS_QUALIFIER=pProcessName and PROCESS_TYPE='WF'); vProcessName := (select MAX(PROCESS_NAME) FROM META_CTL_PROCESS where PROCESS_QUALIFIER=pProcessName and PROCESS_TYPE='WF'); if vProcessID <=0 then vErrorMessage := (select 'Process with name: ' || pProce ssName||' with Process Type of WF not exist in MTA_CTL_PROCESS table'); vErrorMessage := (select 'Error Occured at Step:'|| Step ||':'|| vErrorMessage); -- Print ErrorMessage errorFlag := 1; errorType := 'Process Failure';

RAISE EXCEPTION 'Error: %', vErrorMessage; RETURN -1; end if; Step := '1.11'; -- Print Step||': Getting Target Table Name' vTgtTableName := (select TARGET_TABLE_NAME FROM META_CTL_PROCESS WHERE process_sid = vProcessID); if errorFlag = 0 then Step := '1.1'; vProcessRunID := (select nvl(MAX(process_run_sid),-1) FR OM META_CTL_PROCESS_STATUS WHERE process_sid=vProcessID AND STATUS='RUNNING'); end if; if errorFlag = 0 and vProcessRunID <=0 then vErrorMessage := (select 'Process with name: '||pProcess Name||' is not in RUNNING STATUS in META_CTL_PROCESS_STATUS' ); vErrorMessage := (select 'Error Occured at Step:'|| Step ||':'|| vErrorMessage); errorType := 'Process Failure'; -- Print vErrorMessage RAISE EXCEPTION 'Error: %', vErrorMessage; RETURN -1; errorFlag := 1; end if; end if; Step := '1.2'; -- Print Step||': Verifying Status & Validating Data'||convert(varchar,e rrorFlag) -- Check the stage process even if the Process is Succeeded if errorFlag = 0 then if pProcessStatus='SUCCESS' then errorType := 'Process Failure'; if pProcessType like '%STAGE' then Step := '1.3'; -- Print Step||': Handling when process type is stage' vFileID := (select nvl(MAX(file_sid),-1) FROM META_CTL_FILE FL JO IN META_CTL_FEED FD ON FL.SOURCE_ID=FD.SOURCE_ID where process_run_sid=pP rocessRunID AND ((nvl(FD.IGNORE_ERRO R_RECORDS,'N')='N' AND ERROR_RECORD_COUNT > 0) OR (nvl(IGNORE_COUNT_VARIANCE,'N')='N' AND EXPECTED_FILE_RECORD _COUNT!=(STAGE_RECORD_COUNT+HEADER_RECORD_COUNT))));

if errorFlag = 0 and vFileID > 0 then -- Print 'File Id:'||convert(varchar,vFi leID) -- Print 'Process Run ID:'||convert(varc har,vProcessRunId) UPDATE PS SET STATUS=CASE WHEN (nvl(FD.IGNORE_ERRO R_RECORDS,'N')='N' AND CF.ERROR_RECORD_COUNT!=0) OR (nvl(FD.IGNORE_ERROR_RECORDS ,'N')='N' AND CF.EXPECTED_FILE_RECORD_COUNT!=(CF.STAGE_RECORD_COUNT+nvl(FD.HEADE R_RECORD_COUNT,0))) THEN 'FAILED' ELSE 'SUCCESS' END, STATUS_REASON=CASE WHEN nvl(FD.IGNORE_ER ROR_RECORDS,'N')='N' AND CF.ERROR_RECORD_COUNT > 0 THEN 'Error rows exist in giv en file ..'||CF.FILE_RECEIVED||' please review the attached file for error rows' WHEN nvl(FD.IGNORE_ERROR_RECORDS ,'N')='N' AND CF.EXPECTED_FILE_RECORD_COUNT != (CF.STAGE_RECORD_COUNT+nvl(FD.HEA DER_RECORD_COUNT,0)) THEN 'Control File Count '||CONVERT(VARCHAR,CF.EXPECTED_FIL E_RECORD_COUNT)||' is different from actual loaded records :'||CONVERT(VARCHAR,C F.STAGE_RECORD_COUNT)|| '+ '||CONVERT(VARCHAR,CF.STAGE_RECORD_COUNT) ELSE 'FAILED' END, STATUS_TYPE=CASE WHEN nvl(FD.IGNORE_ERRO R_RECORDS,'N')='N' AND CF.ERROR_RECORD_COUNT > 0 THEN 'Data Validation' WHEN nvl(FD.IGNORE_ERROR_RECORDS ,'N')='N' AND CF.EXPECTED_FILE_RECORD_COUNT != (CF.STAGE_RECORD_COUNT+nvl(FD.HEA DER_RECORD_COUNT,0)) THEN 'Counts Validation' ELSE 'FAILED' END, END_DT=now() FROM META_CTL_PROCESS_STATUS PS JOIN META_CTL_FILE CF ON CF.process_run_ sid=PS.process_run_sid AND CF.file_sid=vFileID AND PS.process_run_sid=vProcessRu nId JOIN META_CTL_FEED FD on CF.SOURCE_ID=FD .SOURCE_ID; toMailUsers := ( SUPPORT_CONTACT FROM META_CTL_FEED FD JOIN META_CTL_FILE FL ON FD.SOURCE_ID=FL.SOURCE_ID and FL.file_sid=vFileID); UPDATE META_CTL_JOB SET JOB_STATUS='FAILED', END_DT=now() WHERE job_sid=pjobID; vErrorMessage := (select STATUS_REASON F ROM META_CTL_PROCESS_STATUS where process_run_sid=vProcessRunId); errorType := (select STATUS_TYPE,vTgtTab leName=P.TARGET_TABLE_NAME FROM MET A_CTL_PROCESS_STATUS PS JOIN MET A_CTL_PROCESS P select INTERNAL_

ON PS.pr ocess_sid=P.process_sid AND PS.p rocess_run_sid=vProcessRunId ); vErrorMessage := (select 'Error Occured at Step:'|| Step ||':'|| vErrorMessage); if errorType = 'Data Validation' then -- Print 'inside '||errorType||' Table:'||vTgtTableName -- 2/2/11 - DdW - commenting thi s section out - not creating a flat file of error records for dell smb. end if; RAISE NOTICE 'Error: %', vErrorMessage; RETURN -1 errorFlag := 1; end if; end if; -- Print convert(varchar,errorFlag)||' at step :'||Step if errorFlag = 0 then Step := '1.4'; -- Print Step||': Updating Status when status is success' UPDATE META_CTL_PROCESS_STATUS SET STATUS=pProcessStatus, STATUS_REASON=pProcessMessage, END_DT=now() WHERE process_run_sid=vProcessRunId; end if; else -- if not success if errorFlag = 0 then UPDATE META_CTL_PROCESS_STATUS SET STATUS=pProcessStatus, STATUS_REASON=pProcessMessage, STATUS_TYPE='Process Failure', END_DT=now() WHERE process_run_sid=vProcessRunId; UPDATE META_CTL_JOB SET JOB_STATUS='FAILED', END_DT=now() WHERE job_sid=pjobID; errorFlag := 1; vErrorMessage := pProcessMessage; errorType := 'Process Failure'; end if; end if; --- Print 'at end'||pProcessType

-----end if;

-----

Print Print Print Print

vProcessRunId pProcessStatus ourinsertCount ourupdateCount

if pProcessType like '%TARGET' then -- Print 'Updating Process Feed Status'||convert(varchar,vProces sRunId) if vTgtTableName <> '' then cmd := (select 'select insertCount = count(*) from '||vT gtTableName|| ' tb join vw_get_pending_files vw on tb. file_sid=vw.file_sid and vw.process_qualifier=''' ||pProcessName||''' where create_job_sid ='|| cast(pjobID as varchar(15)) ); -- Print 'Command'||cmd EXECute immediate cmd; cmd := (select 'select updateCount = count(*) from '|| vTgtTableName||' tb join vw_get_pending_ files vw on tb.file_sid=vw.file_sid and vw.process_qualifier='''|| pProcessName||''' where update_job_sid ! = create_job_sid and update_job_sid='||cast(pjobID as varchar(15)) ); EXECute immediate cmd; END if; -- Print -- Print -- Print -- Print -- Print vProcessRunId pProcessStatus ourinsertCount pProcessName errorFlag

UPDATE ST SET STATUS=CASE WHEN errorFlag=1 THEN 'FAILED' ELSE pProcessStat us END, INSERTED_COUNT=ourinsertCount, UPDATED_COUNT=ourupdateCount, file_sid=vw.file_sid, END_DT=now() FROM META_PROCESS_FEED_STATUS ST JOIN vw_get_pending_files vw on ST.file_sid=vw.file_sid AND vw.PROCESS_QUALIFIER=pProcessName WHERE process_run_sid=vProcessRunId AND STATUS='RUNNING'; end if;

------------ Print 'Error Flag Value:'||convert(varchar,errorFlag) if errorFlag = 1 then UPDATE META_CTL_PROCESS_STATUS

SET

STATUS='FAILED', STATUS_TYPE='Process Failure', STATUS_REASON=vErrorMessage WHERE process_run_sid=vProcessRunId; tableHTML := (select '<H3><font color="#000FF0" font="verdana">E xecution Status Update for Process: '||vProcessName||'</Font></H3>'); tableHTML := (select tableHTML || '<H4>Executed on: '||CAST(now( ) AS VARCHAR(25))||'</H4>' ); tableHTML := (select tableHTML || '<table border="1" font="verda na" color="green">' ); tableHTML := (select tableHTML || '<tr bgcolor="#000FF0"><th>Sta tus Message </th></tr>'); tableHTML := (select tableHTML || '<tr><td align="left" font="ve rdana"><i> '|| vErrorMessage ||'</i></td></tr></table>'); if errorType = 'Process Failure' then toMailUsers := (select nvl(PARAM_VALUE_MASK,'') FROM META_PROCESS_PARAME TER PR JOIN META_GLOBAL_PARAMET ERS P ON P.PARAMETER_sID=PR.PA RAMETER_sID AND PARAMETER_NaMe='$$su pportEmailList' AND PR.process_sid=1); if toMailUsers = '' then toMailUsers := 'ddewald@merkleinc.com'; end if; subMessage := (select 'Process Failure Notification for :'|| pProcessName); else if toMailUsers = '' then set toMailUsers = 'ddewald@merkleinc.com'; end if; subMessage := (select errorType||' Notification for :'|| vProcessName ); end if; errorFlag := 1; exportFile := exportFile; Need to figure out what this is in Netezza... EXEC msdb..sp_send_dbmail recipients=toMailUsers, subject= subMessage, body= tableHTML, body_format = 'HTML', importance = 'High', file_attachments=exportFile RAISE notice 'Error: %', vErrorMessage; return -1; end if;

---------

if errorFlag = 1 then RAISE EXCEPTION 'Error: %', vErrorMessage; return -1; end if; END; END_PROC; =========================================================================== CREATE OR REPLACE PROCEDURE proc_set_job_status(integer) RETURNS integer LANGUAGE NZPLSQL AS BEGIN_PROC DECLARE pJobID ALIAS FOR $1; v_status varchar(100); BEGIN v_status := ''; v_status := (select MAX(STATUS) from META_CTL_PROCESS_STATUS where job_sid=pJobId GROUP BY process_sid HAVING MAX(STATUS)='FAILED'); if v_status = '' then UPDATE META_CTL_JOB SET JOB_STATUS='SUCCESS' where job_sid=pJobI d; else UPDATE META_CTL_JOB SET JOB_STATUS='FAILED' where job_sid=pJobId ; end if; return pJobId; END; END_PROC; =========================================== CREATE TABLE meta_ctl_feed ( source_id character varying(25) NOT NULL, source_nm character varying(75), source_id_grp character varying(50), source_type character varying(100), active_flag character(1), refresh_flag character(1), file_pattern character varying(250), cnt_file_flag character(1), cnt_file_pattern character varying(250), kl_flag character(1) DEFAULT 'N'::"nvarchar", kl_format character varying(10) DEFAULT 'NONE'::"nvarchar", pgp_flag character(1),

zip_type character(1), delimiter character varying(1), field_count integer, header_record_count integer DEFAULT 0, ignore_count_variance character(1), count_variance_threshold numeric(5,2), ignore_error_records character(1), sftp_flag character(1), sftp_site character varying(50), sftp_user character varying(50), sftp_path character varying(50), file_days character varying(50), client_support_contact character varying(500), internal_support_contact character varying(500), source_xtrct_dt timestamp, source_vendor_cd character varying(100), ref_file_name character varying(100), created_user character varying(100), created_dt timestamp, updated_user character varying(100), updated_dt timestamp ) DISTRIBUTE ON (source_id); ========================================== CREATE TABLE meta_ctl_file ( file_sid integer NOT NULL, source_id character varying(50) NOT NULL, file_received character varying(4000), received_date timestamp, expected_file_record_count bigint, stage_record_count bigint, error_record_count bigint, kl_run_flag character(1) NOT NULL DEFAULT 'N'::"nvarchar", kl_run_id character(8), process_flag character(1) NOT NULL DEFAULT 'N'::"nvarchar", drop_flag character(1) NOT NULL DEFAULT 'N'::"nvarchar", dupe_flag character(2) NOT NULL DEFAULT 'N'::"nvarchar", archive_flag character(1) NOT NULL DEFAULT 'N'::"nvarchar", archive_date timestamp, delete_flag character(1) NOT NULL DEFAULT 'N'::"nvarchar", deleted_dt timestamp, not_loaded_flag character(1), data_valid_flag national character(1), process_run_sid integer, inserted_dt timestamp DEFAULT now(), inserted_user character varying(100), updated_dt timestamp NOT NULL DEFAULT now(), updated_user character varying(100) DEFAULT 'ddewald'::"nvarchar" ) DISTRIBUTE ON (file_sid); ======================================== CREATE TABLE meta_ctl_job ( job_sid integer NOT NULL, job_status character varying(20), start_dt date, end_dt date,

executed_by character varying(50), job_group character varying(50) ) DISTRIBUTE ON (job_sid); ============================================= CREATE TABLE meta_global_parameters ( parameter_sid integer NOT NULL, parameter_name character varying(50), parameter_type character varying(50), parameter_desc character varying(250), insert_dt timestamp, inserted_by character varying(50) ) DISTRIBUTE ON (parameter_sid);

======================================================= CREATE TABLE meta_pp_type ( pp_type_sid integer NOT NULL, pp_type_name character varying(25) NOT NULL, pp_format character varying(30), validation_rule character varying(300), conversion_rule character varying(200) ) DISTRIBUTE ON (pp_type_sid); ================================= CREATE TABLE meta_column ( column_sid integer NOT NULL, table_sid integer NOT NULL, pp_type_sid integer NOT NULL, column_name character varying(256) NOT NULL, req_flg character(1) NOT NULL, used_flg character(1) NOT NULL, datatype character varying(24) NOT NULL, datalen integer NOT NULL, preclen integer, cdi_target_column character varying(255) NOT NULL ) DISTRIBUTE ON (column_sid); ====================================================== CREATE TABLE meta_table ( table_sid integer NOT NULL, table_name character varying(255) NOT NULL, table_source character varying(25) NOT NULL, source_id integer NOT NULL, archive_raw integer NOT NULL, archive_stg integer NOT NULL, last_row_id_to_stg integer NOT NULL, last_proc_date timestamp NOT NULL, last_purge_id integer NOT NULL )

DISTRIBUTE ON (table_sid); ========================================= CREATE TABLE meta_ctl_process ( process_sid integer NOT NULL, process_name character varying(50), process_folder character varying(50), process_qualifier character varying(100), process_type character varying(50), etl_process_type character varying(50), process_group character varying(50), job_group character varying(50), target_table_name character varying(255), created_dt timestamp, created_by character varying(50) ) DISTRIBUTE ON (process_sid); ========================================= CREATE TABLE meta_process_parameter ( process_parameter_sid bigint NOT NULL, process_sid integer NOT NULL, parameter_sid integer NOT NULL, param_value_mask character varying(100), param_value_type character varying(100), param_value_desc character varying(255), param_file_type character varying(100), param_indirect_file character varying(100), param_required character varying(1) ) DISTRIBUTE ON (process_parameter_sid); =============================================CREATE OR REPLACE PROCEDURE sp_move _to_stg(character varying(255)) RETURNS boolean LANGUAGE NZPLSQL AS BEGIN_PROC --############################################################################## ################################################ -- Stored Procedure Name : sp_move_to_stg --- Description : creates the dim_person table. --- Input parameter : Root table name with wildcards --- Returns : 0 if successful -255 if any errors --- Called from : Tidal --- Created By : Debbie de Wald -- Date : 4/23/2011 --- Mod log : 08/05/11 DdW correct integer

length check. -08/24/11 DdW Upper the countr y code -09/25/11 Shawn zhang handle the speci al date format -12/21/11 Shawn zhang -the rule: if pp_type_sid = 0 we think the format ion of date is normal, we will handle it defaultly -if pp_type_sid >0 we think the formation of date is handled specially -the modification are as follows: -1. remove the hard code for date check -2. add the rule to handle sepcial numeric such as 1,234,567 -3. remove update stg_job_control set last_run_ date = now(); -4. use the newest error code; -5. add 'smallint' and 'byteint' for int check, ad d 'decimal','float','double','real' for numeric check --############################################################################## ################################################ DECLARE pTable_Name ALIAS FOR $1; pJob_id bigint; vCreate_dt timestamp; vReturn BOOLEAN; rsql_cmd_txt move_to_stage_sql_cmd%ROWTYPE; vTest integer; qs_message varchar; BEGIN pJob_id := (select nvl(max(job_sid)+ 1,1001) from hist_move_to_stage_sql_cmd); vCreate_dt:= (select now()); vReturn := 0; delete from move_to_stage_sql_cmd; ----------------------------------------------------------------------------------------------- Set up working on script to 'freeze' the rows to address, in case loads are running while -- this proc is. ---------------------------------------------------------------------------------------------delete from meta_table_working; insert into meta_table_working select table_sid, table_name, last_row_id_to_stg, 0 as last_row_id_this_run, now() from meta_table where table_name like pTable_name; insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt,

create_dt ) select pJob_id, 0, 'update meta_table_working set last_row_id_this_run = (select nvl(max(r ow_id),0) from raw_' || t.table_name || ') where table_sid =' || cast(table_sid as varchar(20)) ||';', now() from meta_table t where table_name like pTable_name; -- do the sql cmd, save it off, and clear the table for next step FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP; insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; delete from meta_table_working where last_row_id_this_run = 0 or last_row_id_this_run <= last_row_id_to_stg; ----------------------------------------------------------------------------------------------- Check Pre-Processing types ----------------------------------------------------------------------------------------------- 0,1 General - check datatype and datalen ----------------------------------------------------------------------------------------------- integers, bigint ----------------------------------------------------------------------------------------------- find and move into the error table any values which violate the defined rule . insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100004, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(trim(' || c.column_name ||'), ''-,0123456789'')) > 0 and r.row_id > ' || cast(las t_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';' || 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100004,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' ||

'delete from raw_' || t.table_name || ' r where r.row_id in (select row _id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and lower(c.datatype) in ('integer', 'bigint', 'int', 'smallint', 'byteint' ) and lower(c.column_name) not in ('file_id','row_id','record_id','create_dt' ,'create_user') and c.column_name not like 'hyg_%' order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100004); --update stg_job_control --set last_run_date = now(); ----------------------------------------------------------------------------------------------- numeric ----------------------------------------------------------------------------------------------- find and move into the error table any values which violate the defined rule . insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100006, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(trim(' || c.column_name ||'), ''-.,0123456789'')) > 0 and r.row_id > ' || cast(la st_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as v archar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100006,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast( last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row_ id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid

and lower(c.datatype) in ('numeric','decimal','float','double','real') and lower(c.column_name) not in ('file_id','row_id','record_id','create_dt', 'create_user') and c.column_name not like 'hyg_%' order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100006); --update stg_job_control --set last_run_date = now(); --------------------------------------------------------------------------------------------------- Dates ------------------------------------------------------------------------------------------------- find any with invalid characters for date column. insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100005, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(substring(' || c.column_name ||',1,19), ''-:/ 0123456789'')) > 0 and r.row_id > ' || c ast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100005,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row _id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and lower(c.datatype) in ('datetime','timestamp','date') and lower(c.column_name) not in ('file_id','row_id','record_id','create_d t','create_user') and c.column_name not like 'hyg_%' and c.pp_type_sid = 0 order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks

call sp_exec_sql_cmd_pp_check (pJob_id, 100005); --update stg_job_control --set last_run_date = now(); -- find any that aren't the correct length for a datetime insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100050, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(nvl(' || c.column_name || ',''01/01/2000 01:01:01''))) < 19 and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100050,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row _id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and lower(c.datatype) in ('datetime','timestamp') and lower(c.column_name) not in ('file_id','row_id','record_id','create_dt' ,'create_user') and c.column_name not like 'hyg_%' and c.pp_type_sid = 0 order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100050); --update stg_job_control --set last_run_date = now(); -- find any that aren't the correct length for a date insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100051,

'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(nvl(' || c.column_name || ',''01/01/2000''))) != 10 and r.row_id > ' || cast(la st_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100051,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row _id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and lower(c.datatype) = 'date' and lower(c.column_name) not in ('file_id','row_id','record_id','create_dt' ,'create_user') and c.column_name not like 'hyg_%' and c.pp_type_sid = 0 order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100051); --update stg_job_control --set last_run_date = now(); -- find any with invalid characters for date column. insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100052, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where LENGTH(TRIM(TRIM(' || c.column_name ||'), '''||p.validation_rule||''')) > 0 and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100052,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row

_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t INNER JOIN meta_column c ON t.table_sid = c.table_sid INNER JOIN meta_pp_type p ON c.pp_type_sid = p.pp_type_sid where lower(c.datatype) in ('datetime','timestamp','date') and lower(c.column_name) not in ('file_id','row_id','record_id','create_d t','create_user') and c.column_name not like 'hyg_%' and p.validation_rule is not null order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100052); --update stg_job_control --set last_run_date = now(); -- find any that aren't the correct length for a datetime insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100053, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.ta ble_name || ' r where (LENGTH(TRIM(nvl(' || c.column_name || ','''||p.pp_format||'''))) > LENGTH(''' ||p.pp_format ||''') OR CASE WHEN SUBSTR('''|| p.pp_type_name || ''',1,6) = ''hybrid'' THEN LENGTH(TRIM(nvl(' || c.column_name || ','''||p.pp_format||''')) ) < 6 ELSE LENGTH(TRIM(nvl(' || c.column_name || ','''||p.pp_format||'''))) < LENGTH(''' ||p.pp_format ||''') - 2 END ) and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100053,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast (last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where table_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (select row _id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t INNER JOIN meta_column c ON t.table_sid = c.table_sid INNER JOIN meta_pp_type p ON c.pp_type_sid = p.pp_type_sid

where lower(c.datatype) in ('datetime','timestamp','date') and lower(c.column_name) not in ('file_id','row_id','record_id','create_dt' ,'create_user') and c.column_name not like 'hyg_%' and p.pp_format IS NOT NULL order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100053); --update stg_job_control --set last_run_date = now(); ----- find and move into the error table any values which violate the defined ru le. ----insert into move_to_stage_sql_cmd ---( ---job_sid, ---error_code, ---sql_cmd_txt, ---create_dt ---) ----select pJob_id, ---100005, ---'begin insert into wkng_error_event_date select ' || cast(t.tabl e_sid as varchar(10)) || ', r.row_id, to_date(' || ---c.column_name || ') from raw_' || t.table_name || ---' r where r.row_id > ' || cast(last_row_id_to_stg as varchar(20) ) ||'; EXCEPTION WHEN OTHERS THEN NULL; END;', ---now() ----from meta_table t, ---meta_column c ----where t.table_sid = c.table_sid ----and c.datatype in ('datetime', 'timestamp') ----and c.column_name != 'row_id'; --------- call the proc that executes and logs pp checks ----call sp_exec_sql_cmd_pp_check (pJob_id, 100005); -------update stg_job_control ----set last_run_date = now(); ------------------------------------------------------------------------------------------------ data len ----------------------------------------------------------------------------------------------- Dell was unable to send integers in som cases - fix this here (tested that th ey were good numerics first) vTest := 0; vTest := (select 1 from meta_table_working where table_name = 'channel'); if vTest = 1 then update raw_channel set bu_id= cast(cast(cast(bu_id as numeric(19,9)) as integer) as varc har(19)), rptg_bu_id= cast(cast(cast(rptg_bu_id as numeric(19,9)) as intege r) as varchar(19));

end if; vTest := 0; vTest := (select 1 from meta_table_working where table_name = 'order_header'); if vTest = 1 then update raw_order_header set bill_to_business_unit_identifier= cast(cast(cast(bill_to_business _unit_identifier as numeric(19,9)) as integer) as varchar(19)), source_business_unit_identifier= cast(cast(cast(source_business_u nit_identifier as numeric(19,9)) as integer) as varchar(19)), sold_to_business_unit_identifier= cast(cast(cast(sold_to_business _unit_identifier as numeric(19,9)) as integer) as varchar(19)), ship_to_business_unit_identifier= cast(cast(cast(ship_to_business _unit_identifier as numeric(19,9)) as integer) as varchar(19)), end_user_business_unit_identifier= cast(cast(cast(end_user_busine ss_unit_identifier as numeric(19,9)) as integer) as varchar(19)), reference_business_unit_identifier= cast(cast(cast(reference_busi ness_unit_identifier as numeric(19,9)) as integer) as varchar(19)); end if; --vTest := 0; --vTest := (select 1 from meta_table_working where table_name = 'member_person') ; --if vTest = 1 then -update raw_member_person -set contact_id= cast(cast(cast(contact_id as numeric(19,9)) as intege r) as varchar(19)); --end if; insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100007, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where LENGTH(TRIM(' || c.column_name ||')) > ' || cast(c.datalen+c.preclen as varchar(2 0)) || ' and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100007,''' ||c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid= c.table_sid

and lower(c.datatype) not in ('datetime','timestamp','date') and lower(c.datatype) not in ('integer','bigint','smallint','int','n umeric','decimal','real','float', 'double') and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') and c.column_name not like 'hyg_%' --and c.pp_type_sid not in (42) order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100007); --update stg_job_control --set last_run_date = now(); -----------------------------------------------------------------handle int data similar as 12,123,456 ---------------------------------------------------------------insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100007, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where LENGTH(TRIM(TRANSLATE(' || c.column_name ||','','',''''))) > ' || cast(c.datalen+c.preclen as varchar(20)) || ' and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100007,''' ||c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid= c.table_sid and lower(c.datatype) in ('integer','bigint','smallint','int','numer ic','decimal','real','float', 'double') and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') and c.column_name not like 'hyg_%' --and c.pp_type_sid = 42 order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks

call sp_exec_sql_cmd_pp_check (pJob_id, 100007); --update stg_job_control --set last_run_date = now(); -- done datalen ----------------------------------------------------------------------------------------------- required ---------------------------------------------------------------------------------------------insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100008, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where nvl(' || c.column_name ||' ,'''') = '''' and r.row_id > ' || cast(last_ro w_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100008,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and c.req_flg = 'Y' and c.column_name not in ('file_id','row_id','record_id','create_dt' ,'create_user') and c.column_name not like 'hyg_%' order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100008); --update stg_job_control --set last_run_date = now(); ----------------------------------------------------------------------------------------------- Check the Country

-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Dell was unable to send good countries in some cases - but we can't let futur e joins fail - try -- to infer country from member data. --vTest := 0; --vTest := (select 1 from meta_table_working where table_name = 'gold_org'); --if vTest = 1 then --update raw_gold_org go -set country_code = h.country_2char, -create_user = 'infer country' -from dell_smb_b..dim_geo h, -stg_member_org m -where m.org_enterprise_identifier = go.enterprise_id -and nvl(go.country_code,'XX') = 'XX' -and trim(upper(m.source_code),'ABCDEFGHIJKLMNOPQRSTUVWXYZ~!@#$%^&*()_ +-={}|:"<>?`[]\;,./') = cast(h.bu_id as varchar(20)) -and m.row_id = (select max(row_id) -from dell_smb_stage_test..stg_member_org m2 -where m.org_enterprise_identifier= m2.org_enterpris e_identifier); ----mark any remainers as XX -update raw_gold_org -set country_code = 'XX' -where country_code is null; --end if; ---vTest := 0; --vTest := (select 1 from meta_table_working where table_name = 'gold_person'); --if vTest = 1 then --update raw_gold_person go -set country_code = h.country_2char, -create_user = 'infer country' -from dell_smb_b..dim_geo h, -stg_member_person m -where m.person_enterprise_id = go.enterprise_id -and nvl(go.country_code,'XX')= 'XX' -and trim(upper(m.source_code),'ABCDEFGHIJKLMNOPQRSTUVWXYZ~!@#$%^&*()_+ -={}|:"<>?`[]\;,./') = cast(h.bu_id as varchar(20)) -and m.row_id = (select max(row_id) -from dell_smb_stage_test..stg_member_person m2 -where m.person_enterprise_id = m2.person_enterprise_i d); ----mark any remainers as XX -update raw_gold_person -set country_code = 'XX' -where country_code is null --; --end if; ---- some feeds have two addresses - in that case, we want to make sure both are wrong --insert into move_to_stage_sql_cmd

-( -job_sid, -error_code, -sql_cmd_txt, -create_dt -) --select pJob_id, -100001, -'insert into error_' || t.table_name || ' select r.* from raw_ ' || t.table_name || -' r where nvl(r.' || c1.column_name || ',''US'') not in (selec t l.lookup_iso_country_name from lookup_iso_country l) and nvl(r.' || -c2.column_name || ',''US'') not in (select l.lookup_iso_countr y_name from lookup_iso_country l) and r.row_id > ' || -cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ';'|| -'insert into error_event select nvl(d.file_id,0), ' || cast(pJ ob_id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || -', d.row_id, ' || '100001,''' || c1.column_name || -''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || -' and d.row_id not in (select row_id from error_event where ta ble_id = ' || cast(t.table_sid as varchar(20)) || ');' || -'delete from raw_' || t.table_name || ' r where r.row_id in (s elect row_id from error_' || t.table_name || -' e where e.row_id > ' || cast(last_row_id_to_stg as varchar( 20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', -now() --from meta_table_working t, -meta_column c1, -meta_column c2 --where t.table_sid = c1.table_sid --and t.table_sid = c2.table_sid --and c1.pp_type_sid in (2) -- Country --and lower(c1.column_name) not in ('file_id','row_id','record_id',' create_dt','create_user') --and c2.pp_type_sid in (2) -- Country --and lower(c2.column_name) not in ('file_id','row_id','record_id',' create_dt','create_user') --and c1.column_name not like 'hyg_%' --and c2.column_name not like 'hyg_%' --and lower(c1.column_name) != lower(c2.column_name) --; ----- call the proc that executes and logs pp checks --call sp_exec_sql_cmd_pp_check (pJob_id, 100001); ---update stg_job_control --set last_run_date = now(); -- is it a valid country? insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100001,

'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where upper(nvl(r.' || c.column_name || ',''US'')) not in (s elect l.lookup_iso_country_name from lookup_iso_country l) and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || c ast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100001,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and c.pp_type_sid in (2) -- Country and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') --and t.table_sid not in (select table_sid from meta_column where pp _type_sid = 2 group by table_sid having count(table_sid) <2) -- none that have t wo - we checked that already and c.column_name not like 'hyg_%' order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100001); --update stg_job_control --set last_run_date = now(); -- is it a valid long country? insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100001, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where nvl(r.' || c.column_name || ',''United States'') not i n (select l.long_country_name from lookup_iso_country l) and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || c ast(last_row_id_this_run as varchar(20)) || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100001,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl

e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and c.pp_type_sid in (15) -- Country and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') and c.column_name not like 'hyg_%' order by t.table_name, c.column_name; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100001); --update stg_job_control --set last_run_date = now(); -- Can we do business with this country? insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100002, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where upper(nvl(r.' || c.column_name || ',''US'')) in (select l.lookup_iso_country_name from lookup_iso_ country l where can_do_business_with = 0) and r.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and row_id <=' || c ast(last_row_id_this_run as varchar(20)) || ';' || 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100002,''' || c.column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and c.pp_type_sid = 2 -- Country and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') and c.column_name not like 'hyg_%' order by t.table_name, c.column_name;

-- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100002); --update stg_job_control --set last_run_date = now(); ----------------------------------------------------------------------------------------------------------------------- Some rules have conditions - check those rules (required is the only rule tha t is this way) ---------------------------------------------------------------------------------------------------------------------insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 100008, 'insert into error_' || t.table_name || ' select r.* from raw_' || t.table_name || ' r where nvl(' || c.test_column_name ||' ,'''') = '''' and r.row_id > ' || cast(la st_row_id_to_stg as varchar(20)) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ' and ' || c.depend_column_name || ' ' || c.depend_test || ';'|| 'insert into error_event select nvl(d.file_id,0), ' || cast(pJob _id as varchar(20)) || ', ' || cast(t.table_sid as varchar(20)) || ', d.row_id, ' || '100008,''' || c.test_column_name || ''', now() from error_' || table_name || ' d where d.row_id > ' || cast(last_row_id_to_stg as varchar(20)) || ' and d.row_id not in (select row_id from error_event where tabl e_id = ' || cast(t.table_sid as varchar(20)) || ');' || 'delete from raw_' || t.table_name || ' r where r.row_id in (sel ect row_id from error_' || t.table_name || ' e where e.row_id > ' || cast(last_row_id_to_stg as varchar(20 )) || ' and row_id <=' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t, meta_column_depend c where t.table_sid = c.table_sid and c.rule_name = 'required' and c.rule_value = 'Y' and lower(c.test_column_name) not in ('file_id','row_id','record_id' ,'create_dt','create_user') and c.test_column_name not like 'hyg_%' ; -- call the proc that executes and logs pp checks call sp_exec_sql_cmd_pp_check (pJob_id, 100008); --update stg_job_control --set last_run_date = now();

--------------------------------------------------------------------------------------- Email Cleansing process -------------------------------------------------------------------------------------qs_message := 'Email cleansing starts'; insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 0, 'call sp_email_hygiene_process('|| ''''||'raw_' ||t.table_name || '''' ||',' || quote_literal(c.column_name) || ');', now() from meta_table_working t, meta_column c where t.table_sid = c.table_sid and c.pp_type_sid = 14 -- Email address column and lower(c.column_name) not in ('file_id','row_id','record_id','cre ate_dt','create_user') order by t.table_name, c.column_name; -- do the sql cmd, save it off, and clear the table for next step --qs_message := (select sql_cmd_txt from move_to_stage_sql_cmd); FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP; insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; --------------------------------------------------------------------------------------- all done, move the data into the stg and hist tables -------------------------------------------------------------------------------------qs_message := 'Create insert statement starts'; --stg insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 0, 'call sp_create_insert_statement(''' || 'stg' || ''',''' || t.ta ble_name || ''',' || cast(last_row_id_this_run as varchar(20)) || ');',

from

now() meta_table_working t;

-- do the sql cmd, save it off, and clear the table for next step FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP; insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; -- hist insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 0, 'call sp_create_insert_statement(''' || 'hist' || ''',''' || t.t able_name || ''',' || cast(last_row_id_this_run as varchar(20)) || ');', now() from meta_table_working t; -- do the sql cmd, save it off, and clear the table for next step FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP; insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; qs_message := 'delete raw table starts'; -- delete from raw insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 0, 'delete from raw_' || t.table_name || ' r where r.row_id <= ' || cast(last_row_id_this_run as varchar(20)) || ';', now() from meta_table_working t; -- do the sql cmd, save it off, and clear the table for next step FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP;

insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; --------------------------------------------------------------------------------------- Recreate the table with new email hygiene column instead of old email column -------------------------------------------------------------------------------------qs_message := 'Create table statement starts'; insert into move_to_stage_sql_cmd ( job_sid, error_code, sql_cmd_txt, create_dt ) select pJob_id, 0, 'call sp_create_table_statement(' || quote_literal(t.table_name) || ');', now() from meta_table_working t, (select distinct table_sid from meta_column where pp_type_sid=14 ) c where t.table_sid = c.table_sid order by t.table_name; -- do the sql cmd, save it off, and clear the table for next step FOR rsql_cmd_txt IN select * from move_to_stage_sql_cmd LOOP execute immediate rsql_cmd_txt.sql_cmd_txt; END LOOP; insert into hist_move_to_stage_sql_cmd select *,now() from move_to_stage_sql_cmd; delete from move_to_stage_sql_cmd; -- put update set from where in the starting point for the next run meta_table mt last_row_id_to_stg = nvl(last_row_id_this_run,0), last_proc_date = now() meta_table_working l mt.table_sid = l.table_sid;

--delete --from meta_table_working; EXCEPTION WHEN OTHERS THEN RAISE EXCEPTION 'SP FAILED DUE TO ERROR % FOR QUERY %', SQLERRM,qs_message; RETURN false; END; END_PROC;