notebook reading .csv

---------------------------------------------------------------------------

Py4JJavaError                             Traceback (most recent call last)

Cell In[107], line 3

      1 #path = "Files/factor/xyz/year=" + str(yearNo) + "/month=" + monthNo + "/day=" + dayNo + "/bau_raw_v2_1.csv"

      2 path = 'Files/factor/xyz/year=2024/month=08/day=16/bau_raw_v2_1.csv'

----> 3 df = spark.read.csv(path, header=True)

      4 display(df)

 

File /opt/spark/python/lib/pyspark.zip/pyspark/sql/readwriter.py:727, in DataFrameReader.csv(self, path, schema, sep, encoding, quote, escape, comment, header, inferSchema, ignoreLeadingWhiteSpace, ignoreTrailingWhiteSpace, nullValue, nanValue, positiveInf, negativeInf, dateFormat, timestampFormat, maxColumns, maxCharsPerColumn, maxMalformedLogPerPartition, mode, columnNameOfCorruptRecord, multiLine, charToEscapeQuoteEscaping, samplingRatio, enforceSchema, emptyValue, locale, lineSep, pathGlobFilter, recursiveFileLookup, modifiedBefore, modifiedAfter, unescapedQuoteHandling)

    725 if type(path) == list:

    726     assert self._spark._sc._jvm is not None

--> 727     return self._df(self._jreader.csv(self._spark._sc._jvm.PythonUtils.toSeq(path)))

    728 elif isinstance(path, RDD):

    730     def func(iterator):

 

File ~/cluster-env/trident_env/lib/python3.10/site-packages/py4j/java_gateway.py:1322, in JavaMember.__call__(self, *args)

   1316 command = proto.CALL_COMMAND_NAME +\

   1317     self.command_header +\

   1318     args_command +\

   1319     proto.END_COMMAND_PART

   1321 answer = self.gateway_client.send_command(command)

-> 1322 return_value = get_return_value(

   1323     answer, self.gateway_client, self.target_id, self.name)

   1325 for temp_arg in temp_args:

   1326     if hasattr(temp_arg, "_detach"):

 

File /opt/spark/python/lib/pyspark.zip/pyspark/errors/exceptions/captured.py:169, in capture_sql_exception.<locals>.deco(*a, **kw)

    167 def deco(*a: Any, **kw: Any) -> Any:

    168     try:

--> 169         return f(*a, **kw)

    170     except Py4JJavaError as e:

    171         converted = convert_exception(e.java_exception)

 

File ~/cluster-env/trident_env/lib/python3.10/site-packages/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)

    324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)

    325 if answer[1] == REFERENCE_TYPE:

--> 326     raise Py4JJavaError(

    327         "An error occurred while calling {0}{1}{2}.\n".

    328         format(target_id, ".", name), value)

    329 else:

    330     raise Py4JError(

    331         "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".

    332         format(target_id, ".", name, value))