𝘴𝘤𝘳𝘪𝘱𝘵

#! /𝘣𝘪𝘯/𝘣𝘢𝘴𝘩
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [INFO] - main begin"

𝘴𝘦𝘵 +𝘝𝘟
#######################
#check parameters
####################
if [ $# -eq 0 ]
then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - no argument passed"

echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - <arg1> ,<arg2>,<option>"
exit 1

elif [ $# -gt 3 ]
then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - to many argument passed"

echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - <arg1> ,<arg2>,<option>"
exit 1
 fi

######################
#compute the agrs, source name
#####################
ENVARG=`echo "$1" | tr '[:upper:]' '[:lower:]' `
DEV=`echo "$2" | tr '[:upper:]' '[:lower:]' `
OptionVar=`echo "$3" | tr '[:upper:]' '[:lower:]' `

source ~/.password
rc=$? 
if [ $rc -gt 0 ]; then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - Error in sourcing password"
exit 1
fi
#####################
# checking folder exit or not
######################
cd /home/
rc=$? 
if [ $rc -gt 0 ]; then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - no home dir"
exit 1
fi

#####################
# checking file exit or not
######################

rc=$? 
if [ ! - f db.conf ]; then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - check db.conf not found"
exit 1
fi

#####################
# checking option demo.dev exit or not
######################
if [ [  $OptionVar = "dev" ]]
then
      if [ ! - f demo.$OptionVar ]; then
      echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` -               Ingestion - $1 - [Error] - check                              demo.dev not found"
      exit 1
      fi
else
if [ ! - f demo]; then
      echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` -               Ingestion - $1 - [Error] - check                              demo not found"
      exit 1
      fi
#####################
# soucreing dbname, pwd, uname, port
######################
cd /homehome
source db.conf
rc=$? 
if [ $rc -gt 0 ]; then
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Ingestion - $1 - [Error] - soucre db.conf not found"
exit 1
fi

echo $(eval echo \${USERNAME}) 
echo $(eval echo \${PASSWORD}) 
echo $(eval echo \${JDBCURL}) 
#################
getting for properties file 
#################
user_details=$(whoami)
file_path=/home/${user_details}/connection.properties

DB_USER=`grep "mySqlUserName" ${file_path} | cut -d '=' -f 2`
DB_PASSWD=`grep "mySqlPassword" ${file_path} | cut -d '=' -f 2`
DB_PORT=`grep "mySqlPort" ${file_path} | cut -d '=' -f 2`
DB_HOST=`grep "mySqlHostName" ${file_path} | cut -d '=' -f 2`
DB_NAME=`grep "mySqlDBName" ${file_path} | cut -d '=' -f 2`

#######
#Conditions
#####
/*
A.txt
Everyday|dbname|tbname|select * from xyz
*/
Count=1
While IFS= read -r I
do
Loa=`echo $i | cut -d '|' -f1| tr '[upper]' '[lower]' `
Db=`echo $i | cut -d '|' -f3| tr '[upper]' '[lower]' `
Tb=`echo $i | cut -d '|' -f5| tr '[upper]' '[lower]' `
Quer=`echo $i | cut -d '|' -f7| tr '[upper]' '[lower]' `
if [ [ $Loa = "Everyday" ]];then
# here * wil give all location file so use ~
    Cat a.txt | sed -n '5, 40' | sed "s/sometx/$Db/g" | sed "s~quetx~ select * from $Db.$Tb/g" done>>newfile.txt

##key/value
while IFS='=' read -r key value
do
    key=$(echo $key)
    eval ${key}=\${value}
done < "$file"
table_name.txt

elif [ [ $Loa = "montly" ]];then
      
      Count=`expr $count+1`
fi

done < a.txt

#read frm file line by line
#!/bin/bash

echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - Injection - [INFO] - main begin"

while read -r line;
do
   echo "$line" ;
done < table_name.txt

export http_proxy=http://proxy.houston.hpecorp.net:8080
export https_proxy=http://proxy.houston.hpecorp.net:8080
floder_name=eddata

rm -rf $floder_name
echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name folder - [INFO] - removed $floder_name if it is present"

status_code=`curl -s -w "%{http_code}" --output "$floder_name.gz" --data "auth_name=cn%3Dsrv_acct_acct_itgr%2Cou%3DApplications%2Co%3Dhp.com&auth_passwd=aabcpass#123&action=download_std&fileid=1061&compress=1" --url "https://abc/download/download.cgi"`

if [[ "$status_code" -ne 200 ]] ; then
  echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name- [INFO] - Failed with code $status_code"
  exit 0
else
  echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name - [INFO] - Successfull created with status code $status_code"
fi

echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name folder - [INFO] - Created $floder_name is created"
gzip -d $floder_name.gz

check_size=10
file_size=$(du -b $HOME_DIR | cut -f1)
if [ "$file_size" -ge "$check_size" ]; then
   echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name folder - [INFO] - File size is $file_size"
else 
   echo "`date '+%Y-%m-℅d %H:%M:%S.%N' ` - $floder_name folder - [INFO] - File has no record and $file_size is less"
   exit 0
fi

Comments

Post a Comment

Popular posts from this blog

Shell encrypt and decrypt

   #!/bin/sh       # path to virtualenv cd /home/srvc_nextgen_hitg/cp360 source cp360venv/bin/activate echo "Virtualenv started" cd /home/cp360/python_scripts #Read python code export AES_SECRET_KEY="K{;5%A5yHL&^efe-" export apiacc="yen.why.saw-99" encrypt_pswd=`python - <<END import os from AES import AES_ENCRYPT aes_obj = AES_ENCRYPT() encpt = aes_obj.encrypt(os.environ['apiacc'],os.environ['AES_SECRET_KEY']) print(encpt) END`  echo "------encrypt_pswd------------" echo $encrypt_pswd echo "-------------------------------" export encrypt_pswd=$encrypt_pswd decrypt_pswd=`python - <<END import os from AES import AES_ENCRYPT aes_obj = AES_ENCRYPT() dencpt = aes_obj.decrypt(os.environ['encrypt_pswd'],os.environ['AES_SECRET_KEY']) print(dencpt) END`  echo "------decrypt_pswd------------" echo $decrypt_pswd echo "-------------------------------" #############################...
Read more

cdc

 select  CONCAT(TRIM(ASSET.asset_number),0),asset.data_sou) AS cdc_key ,CONCAT(nvl(TRIM(asset.asset_number),'0'),asset.data_sou) AS dat_sou ,md5(concat(nvl(asset.asset_number,'0'),nvl(asset.dat_sou,'0'),nvl(asset.coln,'0') AS cdc_hash ,asset.* from ( select distinct CONCAT(trim(cus_nu),'-'), cast(NULL as string) as vendor_id, ACUR.*, 'N' ISDELETED from INERMIDTE.ACUR )ASSET ===================================== WITH ADDTION AS ( SELECT A.*  ,current_timestamp AS Date_created1, ,current_timestamp AS Date_updated1, ,'N' AS ISDELTED1 FROM precdc.asset A left outer join processed.asset B ON (A.cdc_key = B.cdc_key)  WHERE B.data_sou='1002' AND  (B.cdc_hash IS Null) ) ,DELETION  AS ( SELECT B.* ,B.current_timestamp AS Date_created1, ,B.current_timestamp AS Date_updated1, ,'Y' AS ISDELTED1 FROM precdc.asset A rigt outer join processed.asset B ON (A.cdc_key = B.cdc_key)  WHERE B.data_sou='1002' AND  (B.cdc_hash IS ...
Read more

incrementally update

Incrementally update an imported table In  CDP Private Cloud Base , updating imported tables involves importing incremental changes made to the original table using Sqoop and then merging changes with the tables imported into Hive. After ingesting data from an operational database to Hive, you usually need to set up a process for periodically synchronizing the imported table with the operational database table. The base table is a Hive-managed table that was created during the first data ingestion. Incrementally updating Hive tables from operational database systems involves merging the base table and change records to reflect the latest record set. You create the incremental table as a Hive external table, typically from CSV data in HDFS, to store the change records. This external table contains the changes (INSERTs and UPDATEs) from the operational database since the last data ingestion. Generally, the table is partitioned and only the latest partition is updated, making this pro...
Read more