move html and json io managers
This commit is contained in:
@@ -1,4 +1,4 @@
|
|||||||
from shared.resources import BaseIOManager
|
from shared.io_manager.base import BaseIOManager
|
||||||
from upath import UPath
|
from upath import UPath
|
||||||
|
|
||||||
import dagster as dg
|
import dagster as dg
|
||||||
@@ -1,26 +1,11 @@
|
|||||||
import json
|
import json
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from shared.resources import BaseIOManager
|
from shared.io_manager.base import BaseIOManager
|
||||||
from upath import UPath
|
from upath import UPath
|
||||||
|
|
||||||
import dagster as dg
|
import dagster as dg
|
||||||
|
|
||||||
|
|
||||||
def _process_env_vars(config: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
out = {}
|
|
||||||
for key, value in config.items():
|
|
||||||
if (
|
|
||||||
isinstance(value, dict)
|
|
||||||
and len(value) == 1
|
|
||||||
and next(iter(value.keys())) == "env"
|
|
||||||
):
|
|
||||||
out[key] = dg.EnvVar(next(iter(value.values()))).get_value()
|
|
||||||
else:
|
|
||||||
out[key] = value
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
class JsonIOManager(BaseIOManager):
|
class JsonIOManager(BaseIOManager):
|
||||||
extension: str = ".json" # Automatically adds a .json extension
|
extension: str = ".json" # Automatically adds a .json extension
|
||||||
|
|
||||||
Reference in New Issue
Block a user