https://kedro.org/ logo
Powered by
#questions
Title
# questions
e

Esteban Obando

07/11/2023, 9:37 PM
Copy code
---------------------------------------------------------------------------
Exit                                      Traceback (most recent call last)
File /databricks/python/lib/python3.10/site-packages/click/core.py:1063, in BaseCommand.main(self, args, prog_name, complete_var, standalone_mode, windows_expand_args, **extra)
   1056         # it's not safe to `ctx.exit(rv)` here!
   1057         # note that `rv` may actually contain data like "1" which
   1058         # has obvious effects
   (...)
   1061         # even always obvious that `rv` indicates success/failure
   1062         # by its truthiness/falsiness
-> 1063         ctx.exit()
   1064 except (EOFError, KeyboardInterrupt):

File /databricks/python/lib/python3.10/site-packages/click/core.py:681, in Context.exit(self, code)
    680 """Exits the application with a given exit code."""
--> 681 raise Exit(code)

Exit: 0

During handling of the above exception, another exception occurred:

SystemExit                                Traceback (most recent call last)
    [... skipping hidden 1 frame]

File <command-991559434375355>:41
     39 params_string = ','.join(list_params)
---> 41 main(
     42     [
     43         "--env", conf,
     44         "--pipeline", pipeline,
     45         f"--params={params_string}"
     46     ]
     47 )  # o

File /local_disk0/.ephemeral_nfs/envs/pythonEnv-2abca7aa-105b-47df-bf33-dd90d478d3bc/lib/python3.10/site-packages/encounter_kedro/__main__.py:47, in main(*args, **kwargs)
     46 run = _find_run_command(package_name)
---> 47 run(*args, **kwargs)

File /databricks/python/lib/python3.10/site-packages/click/core.py:1128, in BaseCommand.__call__(self, *args, **kwargs)
   1127 """Alias for :meth:`main`."""
-> 1128 return self.main(*args, **kwargs)

File /databricks/python/lib/python3.10/site-packages/click/core.py:1081, in BaseCommand.main(self, args, prog_name, complete_var, standalone_mode, windows_expand_args, **extra)
   1080 if standalone_mode:
-> 1081     sys.exit(e.exit_code)
   1082 else:
   1083     # in non-standalone mode, return the exit code
   1084     # note that this is only reached if `self.invoke` above raises
   (...)
   1089     # `ctx.exit(1)` and to `return 1`, the caller won't be able to
   1090     # tell the difference between the two

SystemExit: 0

During handling of the above exception, another exception occurred:

AssertionError                            Traceback (most recent call last)
    [... skipping hidden 1 frame]

File /databricks/python/lib/python3.10/site-packages/IPython/core/interactiveshell.py:2047, in InteractiveShell.showtraceback(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)
   2044 if exception_only:
   2045     stb = ['An exception has occurred, use %tb to see '
   2046            'the full traceback.\n']
-> 2047     stb.extend(self.InteractiveTB.get_exception_only(etype,
   2048                                                      value))
   2049 else:
   2050     try:
   2051         # Exception classes can customise their traceback - we
   2052         # use this in IPython.parallel for exceptions occurring
   2053         # in the engines. This should return a list of strings.

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:585, in ListTB.get_exception_only(self, etype, value)
    577 def get_exception_only(self, etype, value):
    578     """Only print the exception type and message, without a traceback.
    579 
    580     Parameters
   (...)
    583     value : exception value
    584     """
--> 585     return ListTB.structured_traceback(self, etype, value)

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:452, in ListTB.structured_traceback(self, etype, evalue, etb, tb_offset, context)
    449     chained_exc_ids.add(id(exception[1]))
    450     chained_exceptions_tb_offset = 0
    451     out_list = (
--> 452         self.structured_traceback(
    453             etype, evalue, (etb, chained_exc_ids),
    454             chained_exceptions_tb_offset, context)
    455         + chained_exception_message
    456         + out_list)
    458 return out_list

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:1118, in AutoFormattedTB.structured_traceback(self, etype, value, tb, tb_offset, number_of_lines_of_context)
   1116 else:
   1117     self.tb = tb
-> 1118 return FormattedTB.structured_traceback(
   1119     self, etype, value, tb, tb_offset, number_of_lines_of_context)

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:1012, in FormattedTB.structured_traceback(self, etype, value, tb, tb_offset, number_of_lines_of_context)
   1009 mode = self.mode
   1010 if mode in self.verbose_modes:
   1011     # Verbose modes need a full traceback
-> 1012     return VerboseTB.structured_traceback(
   1013         self, etype, value, tb, tb_offset, number_of_lines_of_context
   1014     )
   1015 elif mode == 'Minimal':
   1016     return ListTB.get_exception_only(self, etype, value)

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:865, in VerboseTB.structured_traceback(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)
    856 def structured_traceback(
    857     self,
    858     etype: type,
   (...)
    862     number_of_lines_of_context: int = 5,
    863 ):
    864     """Return a nice text document describing the traceback."""
--> 865     formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
    866                                                            tb_offset)
    868     colors = self.Colors  # just a shorthand + quicker name lookup
    869     colorsnormal = colors.Normal  # used a lot

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:799, in VerboseTB.format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)
    796 assert isinstance(tb_offset, int)
    797 head = self.prepare_header(etype, self.long_header)
    798 records = (
--> 799     self.get_records(etb, number_of_lines_of_context, tb_offset) if etb else []
    800 )
    802 frames = []
    803 skipped = 0

File /databricks/python/lib/python3.10/site-packages/IPython/core/ultratb.py:854, in VerboseTB.get_records(self, etb, number_of_lines_of_context, tb_offset)
    848     formatter = None
    849 options = stack_data.Options(
    850     before=before,
    851     after=after,
    852     pygments_formatter=formatter,
    853 )
--> 854 return list(stack_data.FrameInfo.stack_data(etb, options=options))[tb_offset:]

File /databricks/python/lib/python3.10/site-packages/stack_data/core.py:578, in FrameInfo.stack_data(cls, frame_or_tb, options, collapse_repeated_frames)
    562 @classmethod
    563 def stack_data(
    564         cls,
   (...)
    568         collapse_repeated_frames: bool = True
    569 ) -> Iterator[Union['FrameInfo', RepeatedFrames]]:
    570     """
    571     An iterator of FrameInfo and RepeatedFrames objects representing
    572     a full traceback or stack. Similar consecutive frames are collapsed into RepeatedFrames
   (...)
    576     and optionally an Options object to configure.
    577     """
--> 578     stack = list(iter_stack(frame_or_tb))
    580     # Reverse the stack from a frame so that it's in the same order
    581     # as the order from a traceback, which is the order of a printed
    582     # traceback when read top to bottom (most recent call last)
    583     if is_frame(frame_or_tb):

File /databricks/python/lib/python3.10/site-packages/stack_data/utils.py:97, in iter_stack(frame_or_tb)
     95 while frame_or_tb:
     96     yield frame_or_tb
---> 97     if is_frame(frame_or_tb):
     98         frame_or_tb = frame_or_tb.f_back
     99     else:

File /databricks/python/lib/python3.10/site-packages/stack_data/utils.py:90, in is_frame(frame_or_tb)
     89 def is_frame(frame_or_tb: Union[FrameType, TracebackType]) -> bool:
---> 90     assert_(isinstance(frame_or_tb, (types.FrameType, types.TracebackType)))
     91     return isinstance(frame_or_tb, (types.FrameType,))

File /databricks/python/lib/python3.10/site-packages/stack_data/utils.py:176, in assert_(condition, error)
    174 if isinstance(error, str):
    175     error = AssertionError(error)
--> 176 raise error
d

Deepyaman Datta

07/11/2023, 11:56 PM
Not really familiar with it, but seems to be an issue with how the notebook handles exit code? https://stackoverflow.com/questions/72954293/implement-sys-exit-in-databricks-python-activity looks to have a similar error message (no solution there, just sharing) Can you not just put the 
main()
call inside a try/except block and catch 
Exit
?
(and use the exit code to determine whether it failed or not, from the caught exception)
j

Juan Luis

07/12/2023, 9:45 AM
continuation of an internal conversation: this is a known issue https://github.com/kedro-org/kedro/issues/1807 and the workaround is to not use the CLI functions from Databricks, but instead run the Kedro session directly, for example running 
%load_ext kedro.ipython
and then 
session.run()
, as explained here: https://docs.kedro.org/en/stable/deployment/databricks/databricks_notebooks_development_workflow.html or, alternatively bootstrap the session manually
Copy code
from kedro.framework.session import KedroSession
from kedro.framework.project import configure_project
package_name = <your_package_name>
configure_project(package_name)

with KedroSession.create(package_name) as session:
    session.run()
@Esteban Obando has mentioned that their Kedro project is packaged as a 
.whl
which makes the bootstrapping difficult. could you detail your setup a bit more?
e

Esteban Obando

07/12/2023, 4:27 PM
the process I'm using to execute it you mean? Currently we are using the main method of the library to execute it within a databricks job
here is the sample
Copy code
from encounter_kedro.__main__ import main

main(
     [
         "--env", conf,
         "--pipeline", pipeline,
         f"--params={params_string}"
     ]
)  # o
the project is in kedro 0.18.7
d

Deepyaman Datta

07/12/2023, 4:37 PM
Currently we are using the main method of the library to execute it within a databricks job
@Juan Luis this is why I thought maybe can just catch the 
Exit
? not sure if I'm missing something
j

Juan Luis

07/12/2023, 9:59 PM
that could be a workaround yes, although the blessed way to do it would be
Copy code
from kedro.framework.session import KedroSession
from kedro.framework.project import configure_project
package_name = 'encounter_kedro'
configure_project(package_name)

with KedroSession.create(package_name) as session:
    session.run()
👍 1
@Esteban Obando could you try this instead?
g

giphy

07/12/2023, 10:00 PM
blessed
😄 1
e

Esteban Obando

07/12/2023, 10:06 PM
ok let me try that
2 Views
Powered by