Hi Good morning Am using Copy activity copy the data from snowflake to lakehouse.
Snowflake structure
create or replace TABLE DB.TABLES.PRODUCT (
PRODUCTID VARCHAR(50),
PRODUCTNAME VARCHAR(100),
CATEGORY VARCHAR(50),
PRICE VARCHAR(50),
STOCKQUANTITY VARCHAR(50),
SUPPLIERID VARCHAR(50),
INSERTEDDATETIME VARCHAR(50),
UPDATEDDATETIME VARCHAR(50),
ISCURRENT BOOLEAN,
LOADDATETIME VARCHAR(50),
AD_INSERTDATETIME VARCHAR(50)
);
Lakehouse table sturucture
schema = StructType([
StructField("PRODUCTID", StringType(), True),
StructField("PRODUCTNAME", StringType(), True),
StructField("CATEGORY", StringType(), True),
StructField("PRICE", StringType(), True),
StructField("STOCKQUANTITY", StringType(), True),
StructField("SUPPLIERID", StringType(), True),
StructField("INSERTEDDATETIME", StringType(), True),
StructField("UPDATEDDATETIME", StringType(), True),
StructField("ISCURRENT", StringType(), True),
StructField("HASHKEY", StringType(), True),
StructField("AD_INSERTDATETIME", StringType(), True),
])
Question 1
1. i create new table in in lakehouse using pyspark,only schema datatype StringType(), copy the data from snowflake to lakehouse,if i change the datatype empty dataset rows copied to lakehouse.is there anything have to do ?
Question 2
2. I need to create a new hashkey
{
"type": "TabularTranslator",
"mappings": [
{
"source": { "name": "AD_INSERTDATETIME" },
"sink": { "name": "AD_INSERTDATETIME" }
},
{
"source": { "name": "ISCURRENT" },
"sink": { "name": "ISCURRENT" }
},
{
"source": { "name": "UPDATEDDATETIME" },
"sink": { "name": "UPDATEDDATETIME" }
},
{
"source": { "name": "INSERTEDDATETIME" },
"sink": { "name": "INSERTEDDATETIME" }
},
{
"source": { "name": "SUPPLIERID" },
"sink": { "name": "SUPPLIERID" }
},
{
"source": { "name": "STOCKQUANTITY" },
"sink": { "name": "STOCKQUANTITY" }
},
{
"source": { "name": "PRICE" },
"sink": { "name": "PRICE" }
},
{
"source": { "name": "CATEGORY" },
"sink": { "name": "CATEGORY" }
},
{
"source": { "name": "PRODUCTNAME" },
"sink": { "name": "PRODUCTNAME" }
},
{
"source": { "name": "PRODUCTID" },
"sink": { "name": "PRODUCTID" }
},
{
"source": { "name": "LOADDATETIME" },
"sink": { "name": "LOADDATETIME" }
},
{
"source": {
"name": "HASHKEY",
"expression": "HASH(CONCAT(PRODUCTID, PRODUCTNAME))"
},
"sink": {
"name": "HASHKEY"
}
}
]
}
but its not work for me. if anything miss kindly let me know fix the above 2 isssues ?
