Forum Discussion
sun-sboyanapall
Advocate I
1 year agoCustom Library not found
Hello, I have compiled a whl file and placed it unde rthe custom library section and published it and when i try to import it under notebook using the environment which has this library it says ...
- 1 year ago
I have rebulished it with a different name it worked this time.
Here is an example library for refreshing SQL Analytics End Pointimport os import shutil import subprocess # Recreate the directory structure from scratch source_dir = "/mnt/data/lakehouse_sync_new" dist_dir = os.path.join(source_dir, "dist") if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(os.path.join(source_dir, "lakehouse_sync_new")) # Create an empty __init__.py file with open(os.path.join(source_dir, "lakehouse_sync_new", "__init__.py"), "w") as f: f.write("") # Write the updated script without logging script_content_no_logging = """ import json import time from sempy.fabric import FabricRestClient from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException class LakehouseSync: def __init__(self, tenant_id, workspace_id, lakehouse_id): self.tenant_id = tenant_id self.workspace_id = workspace_id self.lakehouse_id = lakehouse_id self.client = FabricRestClient() self.max_retries = 60 # Timeout after 60 seconds self.sleep_duration = 1 # Sleep 1 second between retries def _make_get_request(self, uri): \"\"\"Helper to make a GET request and return JSON data.\"\"\" response = self.client.get(uri) response.raise_for_status() return response.json() def _make_post_request(self, uri, payload): \"\"\"Helper to make a POST request and return JSON data.\"\"\" response = self.client.post(uri, json=payload) response.raise_for_status() return response.json() def get_sql_endpoint(self): \"\"\"Retrieve the SQL endpoint ID for the lakehouse.\"\"\" uri = f"/v1/workspaces/{self.workspace_id}/lakehouses/{self.lakehouse_id}" data = self._make_get_request(uri) return data['properties']['sqlEndpointProperties']['id'] def sync_metadata(self, sql_endpoint_id): \"\"\"Initiate metadata synchronization and monitor progress.\"\"\" sync_uri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}" payload = {"commands": [{"$type": "MetadataRefreshExternalCommand"}]} # Start the sync response = self._make_post_request(sync_uri, payload) batch_id = response["batchId"] progress_state = response["progressState"] # Monitor sync progress status_uri = f"/v1.0/myorg/lhdatamarts/{sql_endpoint_id}/batches/{batch_id}" retries = 0 while progress_state == 'inProgress' and retries < self.max_retries: time.sleep(self.sleep_duration) retries += 1 status_response = self._make_get_request(status_uri) progress_state = status_response["progressState"] if retries >= self.max_retries: raise TimeoutError("Sync process timed out.") if progress_state == 'failure': raise Exception(f"Sync failed. Details: {status_response}") def run(self): \"\"\"Main method to orchestrate the synchronization process.\"\"\" sql_endpoint_id = self.get_sql_endpoint() self.sync_metadata(sql_endpoint_id) """ with open(os.path.join(source_dir, "lakehouse_sync_new", "lakehouse_sync_new.py"), "w") as f: f.write(script_content_no_logging) # Create a minimal setup.py file setup_content_minimal = """ from setuptools import setup, find_packages setup( name="lakehouse_sync_new", version="1.0.0", packages=find_packages(), author="Sai Boyanapalli", ) """ with open(os.path.join(source_dir, "setup.py"), "w") as f: f.write(setup_content_minimal) # Rebuild the .whl file subprocess.run(["python3", "setup.py", "bdist_wheel"], cwd=source_dir, capture_output=True, text=True)
govindarajan_d
Super User
1 year agoHi sun-sboyanapall,
Can you try republishing once? I have previously used a custom wheel in spark and faced no issues.
Also, can you try installing in a virtual environment in your local machine to check if the module gets correctly installed and imports without issue?