Bernardo Branco
05/15/2023, 7:36 PMDeepyaman Datta
05/15/2023, 7:39 PMBernardo Branco
05/15/2023, 7:47 PM@pytest.fixture
def config_loader():
return ConfigLoader(conf_source=conf_path)
@pytest.fixture
def project_context(config_loader):
return KedroContext(
package_name="cnx",
project_path=Path.cwd(),
config_loader=config_loader,
hook_manager=_create_hook_manager(),
)
@pytest.fixture
@hook_impl
def after_context_created(project_context) -> None:
"""Initialises a SparkSession using the config
defined in project's conf folder.
"""
# Load the spark configuration in spark.yaml using the config loader
parameters = project_context.config_loader.get("spark*", "spark*/**")
spark_conf = SparkConf().setAll(parameters.items())
# Initialise the spark session
spark_session_conf = (
#SparkSession.builder.appName(context._package_name)
SparkSession.builder.appName(project_context.project_path.name)
.enableHiveSupport()
.config(conf=spark_conf)
)
_spark_session = spark_session_conf.getOrCreate()
_spark_session.sparkContext.setLogLevel("WARN")
Deepyaman Datta
05/16/2023, 5:48 AMpytest
? @hook_impl
isn't going to do anything here (unless I suppose you're doing something more advanced that I'm not familiar with using pytest's plugin framework, since that is technically what Kedro uses for hooks). You don't have your fixtures for auto-use here, so are you passing the fixture to each function?
In short, your fixture should probably work something like https://www.mikulskibartosz.name/use-one-spark-session-to-run-all-pytest-tests/, but you can load the Kedro config here instead if you'd like when constructing the session.Bernardo Branco
05/16/2023, 10:34 AMNok Lam Chan
05/16/2023, 3:08 PMBernardo Branco
05/16/2023, 3:13 PMNok Lam Chan
05/16/2023, 3:26 PMBernardo Branco
05/16/2023, 3:29 PMRichard Purvis
05/16/2023, 3:52 PMNok Lam Chan
05/16/2023, 4:24 PMbootstrap_project
call.
bootstrap_project(Path("<project_root>"))
with KedroSession.create() as session:
session.run(<some_pipeline>)
https://docs.kedro.org/en/stable/kedro.framework.session.session.KedroSession.html#kedro-framework-session-session-kedrosession