Forum Discussion

sun-sboyanapall's avatar
sun-sboyanapall
Icon for Advocate I rankAdvocate I
1 year ago
Solved

Custom Library not found

Hello,    I have compiled a whl file and placed it unde rthe custom library section and published it and when i try to import it under notebook using the environment which has this library it says ...
  • sun-sboyanapall's avatar
    1 year ago

    I have rebulished it with a different name it worked this time. 

    Here is an example library for refreshing SQL Analytics End Point

    import os
    import shutil
    import subprocess
    
    # Recreate the directory structure from scratch
    source_dir = "/mnt/data/lakehouse_sync_new"
    dist_dir = os.path.join(source_dir, "dist")
    
    if os.path.exists(source_dir):
        shutil.rmtree(source_dir)
    os.makedirs(os.path.join(source_dir, "lakehouse_sync_new"))
    
    # Create an empty __init__.py file
    with open(os.path.join(source_dir, "lakehouse_sync_new", "__init__.py"), "w") as f:
        f.write("")
    
    # Write the updated script without logging
    script_content_no_logging = """
    import json
    import time
    from sempy.fabric import FabricRestClient
    from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException
    
    class LakehouseSync:
        def __init__(self, tenant_id, workspace_id, lakehouse_id):
            self.tenant_id = tenant_id
            self.workspace_id = workspace_id
            self.lakehouse_id = lakehouse_id
            self.client = FabricRestClient()
            self.max_retries = 60  # Timeout after 60 seconds
            self.sleep_duration = 1  # Sleep 1 second between retries
    
        def _make_get_request(self, uri):
            \"\"\"Helper to make a GET request and return JSON data.\"\"\"
            response = self.client.get(uri)
            response.raise_for_status()
            return response.json()
    
        def _make_post_request(self, uri, payload):
            \"\"\"Helper to make a POST request and return JSON data.\"\"\"
            response = self.client.post(uri, json=payload)
            response.raise_for_status()
            return response.json()
    
        def get_sql_endpoint(self):
            \"\"\"Retrieve the SQL endpoint ID for the lakehouse.\"\"\"
            uri = f"/v1/workspaces/{self.workspace_id}/lakehouses/{self.lakehouse_id}"
            data = self._make_get_request(uri)
            return data['properties']['sqlEndpointProperties']['id']
    
        def sync_metadata(self, sql_endpoint_id):
            \"\"\"Initiate metadata synchronization and monitor progress.\"\"\"
            sync_uri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}"
            payload = {"commands": [{"$type": "MetadataRefreshExternalCommand"}]}
    
            # Start the sync
            response = self._make_post_request(sync_uri, payload)
            batch_id = response["batchId"]
            progress_state = response["progressState"]
    
            # Monitor sync progress
            status_uri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}/batches/{batch_id}"
            retries = 0
    
            while progress_state == 'inProgress' and retries < self.max_retries:
                time.sleep(self.sleep_duration)
                retries += 1
                status_response = self._make_get_request(status_uri)
                progress_state = status_response["progressState"]
    
            if retries >= self.max_retries:
                raise TimeoutError("Sync process timed out.")
    
            if progress_state == 'failure':
                raise Exception(f"Sync failed. Details: {status_response}")
    
        def run(self):
            \"\"\"Main method to orchestrate the synchronization process.\"\"\"
            sql_endpoint_id = self.get_sql_endpoint()
            self.sync_metadata(sql_endpoint_id)
    """
    with open(os.path.join(source_dir, "lakehouse_sync_new", "lakehouse_sync_new.py"), "w") as f:
        f.write(script_content_no_logging)
    
    # Create a minimal setup.py file
    setup_content_minimal = """
    from setuptools import setup, find_packages
    
    setup(
        name="lakehouse_sync_new",
        version="1.0.0",
        packages=find_packages(),
        author="Sai Boyanapalli",
    )
    """
    with open(os.path.join(source_dir, "setup.py"), "w") as f:
        f.write(setup_content_minimal)
    
    # Rebuild the .whl file
    subprocess.run(["python3", "setup.py", "bdist_wheel"], cwd=source_dir, capture_output=True, text=True)