API: preprocessing

The preprocessing package contains Calculator/Applier nodes and the FeatureEngineer orchestrator.

skyulf.preprocessing

Preprocessing nodes: one Calculator/Applier pair per pipeline step.

Importing this package re-exports the public node surface listed in __all__ and, as a side effect, runs every @NodeRegistry.register decorator — which is why the imports below are explicit rather than auto-discovered.

AuditedFoldPreprocessor

Decorates a :class:FoldPreprocessor to record per-fold row counts.

Every fit_transform/transform call records the number of input rows it received. The isolation invariant a leak-free run must satisfy is max(fit_rows) <= train_rows — a leaked fit would see the train split plus held-out rows. Exposed via :meth:summary so the app can log it and persist it in node metrics for post-hoc audit (findings 2026-08-26 §3/B).

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

class AuditedFoldPreprocessor:
    """Decorates a :class:`FoldPreprocessor` to record per-fold row counts.

    Every ``fit_transform``/``transform`` call records the number of input
    rows it received. The isolation invariant a leak-free run must satisfy is
    ``max(fit_rows) <= train_rows`` — a leaked fit would see the train split
    plus held-out rows. Exposed via :meth:`summary` so the app can log it and
    persist it in node metrics for post-hoc audit (findings 2026-08-26 §3/B).
    """

    def __init__(self, inner: Any):
        """Wrap ``inner`` and start both row-count logs empty.

        ``changes_row_count`` is mirrored from ``inner`` when it declares one
        and defaults to ``False``, keeping the decorator transparent to callers
        that inspect the flag.
        """
        self._inner = inner
        self.fit_rows: list[int] = []
        self.transform_rows: list[int] = []
        self.changes_row_count = getattr(inner, "changes_row_count", False)

    @property
    def inner(self) -> Any:
        """Return the wrapped preprocessor."""
        return self._inner

    def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
        """Log the fit-time input row count, then delegate to ``inner``."""
        self.fit_rows.append(frame_rows(X))

        X, y, self.train_sample_weight_ = fit_preprocessor(self._inner, X, y, sample_weight)
        return X, y

    def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
        """Log the transform-time input row count, then delegate to ``inner``."""
        self.transform_rows.append(frame_rows(X))
        result = self._inner.transform(X, y)
        self.last_transform_coverage_ = deepcopy(
            getattr(self._inner, "last_transform_coverage_", {})
        )
        return result

    def original_target_labels(self, labels: Any) -> Any:
        """Forward optional fitted target decoding without inferring label identities."""
        decode = getattr(self._inner, "original_target_labels", None)
        return decode(labels) if decode is not None else None

    def transform_tracking_order(self, X: Any) -> tuple[Any, Any]:
        """Record prediction transforms while retaining optional positional lineage."""
        self.transform_rows.append(frame_rows(X))
        transform = getattr(self._inner, "transform_tracking_order", None)
        if transform is not None:
            return transform(X)
        transformed, _ = self._inner.transform(X, None)
        return transformed, None

    def summary(self, train_rows: int | None = None) -> dict[str, Any]:
        """Return per-fold call counts and the largest fit payload seen.

        ``isolation_ok`` appears only when ``train_rows`` is supplied; omitting
        it leaves the invariant unevaluated rather than assumed satisfied.
        """
        result: dict[str, Any] = {
            "fit_calls": len(self.fit_rows),
            "max_fit_rows": max(self.fit_rows, default=0),
            "transform_calls": len(self.transform_rows),
        }
        if train_rows is not None:
            result["train_rows"] = train_rows
            result["isolation_ok"] = result["max_fit_rows"] <= train_rows
        return result

inner property

Return the wrapped preprocessor.

__init__(inner)

Wrap inner and start both row-count logs empty.

changes_row_count is mirrored from inner when it declares one and defaults to False, keeping the decorator transparent to callers that inspect the flag.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def __init__(self, inner: Any):
    """Wrap ``inner`` and start both row-count logs empty.

    ``changes_row_count`` is mirrored from ``inner`` when it declares one
    and defaults to ``False``, keeping the decorator transparent to callers
    that inspect the flag.
    """
    self._inner = inner
    self.fit_rows: list[int] = []
    self.transform_rows: list[int] = []
    self.changes_row_count = getattr(inner, "changes_row_count", False)

fit_transform(X, y, *, sample_weight=None)

Log the fit-time input row count, then delegate to inner.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
    """Log the fit-time input row count, then delegate to ``inner``."""
    self.fit_rows.append(frame_rows(X))

    X, y, self.train_sample_weight_ = fit_preprocessor(self._inner, X, y, sample_weight)
    return X, y

original_target_labels(labels)

Forward optional fitted target decoding without inferring label identities.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def original_target_labels(self, labels: Any) -> Any:
    """Forward optional fitted target decoding without inferring label identities."""
    decode = getattr(self._inner, "original_target_labels", None)
    return decode(labels) if decode is not None else None

summary(train_rows=None)

Return per-fold call counts and the largest fit payload seen.

isolation_ok appears only when train_rows is supplied; omitting it leaves the invariant unevaluated rather than assumed satisfied.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def summary(self, train_rows: int | None = None) -> dict[str, Any]:
    """Return per-fold call counts and the largest fit payload seen.

    ``isolation_ok`` appears only when ``train_rows`` is supplied; omitting
    it leaves the invariant unevaluated rather than assumed satisfied.
    """
    result: dict[str, Any] = {
        "fit_calls": len(self.fit_rows),
        "max_fit_rows": max(self.fit_rows, default=0),
        "transform_calls": len(self.transform_rows),
    }
    if train_rows is not None:
        result["train_rows"] = train_rows
        result["isolation_ok"] = result["max_fit_rows"] <= train_rows
    return result

transform(X, y)

Log the transform-time input row count, then delegate to inner.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
    """Log the transform-time input row count, then delegate to ``inner``."""
    self.transform_rows.append(frame_rows(X))
    result = self._inner.transform(X, y)
    self.last_transform_coverage_ = deepcopy(
        getattr(self._inner, "last_transform_coverage_", {})
    )
    return result

transform_tracking_order(X)

Record prediction transforms while retaining optional positional lineage.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def transform_tracking_order(self, X: Any) -> tuple[Any, Any]:
    """Record prediction transforms while retaining optional positional lineage."""
    self.transform_rows.append(frame_rows(X))
    transform = getattr(self._inner, "transform_tracking_order", None)
    if transform is not None:
        return transform(X)
    transformed, _ = self._inner.transform(X, None)
    return transformed, None

BaseApplier

Bases: ABC

Transforming half of a node pair: applies a fitted artifact to data.

Subclasses implement :meth:apply, taking everything they need from params so the same applier instance can be reused across splits and datasets without carrying state.

Source code in skyulf-core/skyulf/preprocessing/base.py

class BaseApplier(ABC):
    """Transforming half of a node pair: applies a fitted artifact to data.

    Subclasses implement :meth:`apply`, taking everything they need from
    ``params`` so the same applier instance can be reused across splits and
    datasets without carrying state.
    """

    @abstractmethod
    def apply(self, df: pd.DataFrame | SkyulfDataFrame | tuple, params: dict[str, Any]) -> Any:
        """Applies the transformation using fitted parameters.

        The return type is intentionally `Any` because the concrete shape
        depends on the input: passing a `DataFrame` returns a `DataFrame`;
        passing an `(X, y)` tuple returns a tuple; splitters return
        `SplitDataset`. Encoding every case as a union forces callers to
        defensively narrow on every use, which is worse than `Any` here.
        """

apply(df, params) abstractmethod

Applies the transformation using fitted parameters.

The return type is intentionally Any because the concrete shape depends on the input: passing a DataFrame returns a DataFrame; passing an (X, y) tuple returns a tuple; splitters return SplitDataset. Encoding every case as a union forces callers to defensively narrow on every use, which is worse than Any here.

Source code in skyulf-core/skyulf/preprocessing/base.py

@abstractmethod
def apply(self, df: pd.DataFrame | SkyulfDataFrame | tuple, params: dict[str, Any]) -> Any:
    """Applies the transformation using fitted parameters.

    The return type is intentionally `Any` because the concrete shape
    depends on the input: passing a `DataFrame` returns a `DataFrame`;
    passing an `(X, y)` tuple returns a tuple; splitters return
    `SplitDataset`. Encoding every case as a union forces callers to
    defensively narrow on every use, which is worse than `Any` here.
    """

BaseCalculator

Bases: ABC

Learning half of a node pair: derives parameters and returns them as an artifact.

Subclasses implement :meth:fit and hand back a plain mapping that the matching :class:BaseApplier consumes. Calculators never transform data and hold no state between calls.

Source code in skyulf-core/skyulf/preprocessing/base.py

class BaseCalculator(ABC):
    """Learning half of a node pair: derives parameters and returns them as an artifact.

    Subclasses implement :meth:`fit` and hand back a plain mapping that the
    matching :class:`BaseApplier` consumes. Calculators never transform data
    and hold no state between calls.
    """

    @abstractmethod
    def fit(
        self, df: pd.DataFrame | SkyulfDataFrame | tuple, config: dict[str, Any]
    ) -> Mapping[str, Any]:
        """Calculates parameters from the training data.

        Returns a Mapping of fitted parameters (typically a TypedDict
        ``*Artifact`` declared in ``preprocessing._artifacts``). The return
        type is ``Mapping`` rather than ``Dict`` so concrete TypedDict
        subclasses are valid LSP-substitutable returns.
        """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Best-effort prediction of the output schema from config alone.

        Override this in concrete Calculators when the output columns/dtypes
        can be derived purely from ``input_schema`` and ``config`` (i.e.
        without seeing data). Examples:

        * Scalers — preserve column names and predict output dtype promotion.
        * Drop columns by name — drop the configured names.
        * One-hot — adds K columns per categorical (K is data-dependent →
          return ``None``).

        Default returns ``None`` to signal "unknown / data-dependent";
        callers should fall back to runtime introspection.
        """
        return None

fit(df, config) abstractmethod

Calculates parameters from the training data.

Returns a Mapping of fitted parameters (typically a TypedDict *Artifact declared in preprocessing._artifacts). The return type is Mapping rather than Dict so concrete TypedDict subclasses are valid LSP-substitutable returns.

Source code in skyulf-core/skyulf/preprocessing/base.py

@abstractmethod
def fit(
    self, df: pd.DataFrame | SkyulfDataFrame | tuple, config: dict[str, Any]
) -> Mapping[str, Any]:
    """Calculates parameters from the training data.

    Returns a Mapping of fitted parameters (typically a TypedDict
    ``*Artifact`` declared in ``preprocessing._artifacts``). The return
    type is ``Mapping`` rather than ``Dict`` so concrete TypedDict
    subclasses are valid LSP-substitutable returns.
    """

infer_output_schema(input_schema, config)

Best-effort prediction of the output schema from config alone.

Override this in concrete Calculators when the output columns/dtypes can be derived purely from input_schema and config (i.e. without seeing data). Examples:

• Scalers — preserve column names and predict output dtype promotion.
• Drop columns by name — drop the configured names.
• One-hot — adds K columns per categorical (K is data-dependent → return None).

Default returns None to signal "unknown / data-dependent"; callers should fall back to runtime introspection.

Source code in skyulf-core/skyulf/preprocessing/base.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Best-effort prediction of the output schema from config alone.

    Override this in concrete Calculators when the output columns/dtypes
    can be derived purely from ``input_schema`` and ``config`` (i.e.
    without seeing data). Examples:

    * Scalers — preserve column names and predict output dtype promotion.
    * Drop columns by name — drop the configured names.
    * One-hot — adds K columns per categorical (K is data-dependent →
      return ``None``).

    Default returns ``None`` to signal "unknown / data-dependent";
    callers should fall back to runtime introspection.
    """
    return None

CastingApplier

Bases: BaseApplier

Cast each column named in type_map to its target dtype on the active engine.

coerce_on_error=True (the default) is best-effort per column: the pandas path swallows any cast failure and leaves that column untouched, the polars path casts with strict=False. With it off, both engines raise. The awkward families are hand-written on each path because the native casts disagree — polars truncates fractional floats where pandas masks them, and pandas' astype clamps out-of-range integers where polars nulls or raises. Boolean targets additionally go through the shared alias table so "yes"/"no"-style strings coerce the same way on both engines in best-effort mode, including Polars Categorical and Enum labels. Text-to-datetime casts share the pandas mixed-format parser and retain the resolved engine dtype. Columns missing from the frame are skipped. New categorical artifacts freeze the training vocabulary; unseen values become missing. Older artifacts without a vocabulary retain their original casting behavior.

Source code in skyulf-core/skyulf/preprocessing/casting.py

class CastingApplier(BaseApplier):
    """Cast each column named in ``type_map`` to its target dtype on the active engine.

    ``coerce_on_error=True`` (the default) is best-effort per column: the pandas
    path swallows any cast failure and leaves that column untouched, the polars
    path casts with ``strict=False``. With it off, both engines raise. The
    awkward families are hand-written on each path because the native casts
    disagree — polars truncates fractional floats where pandas masks them, and
    pandas' ``astype`` clamps out-of-range integers where polars nulls or raises.
    Boolean targets additionally go through the shared alias table so
    "yes"/"no"-style strings coerce the same way on both engines in best-effort
    mode, including Polars Categorical and Enum labels. Text-to-datetime casts
    share the pandas mixed-format parser and retain the resolved engine dtype.
    Columns missing from the frame are skipped. New categorical artifacts
    freeze the training vocabulary; unseen values become missing. Older
    artifacts without a vocabulary retain their original casting behavior.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect local dtype/vocabulary state without converting or relearning it."""
        fields = {"type", "type_map", "coerce_on_error"}
        if isinstance(raw, dict) and "categories" in raw:
            fields.add("categories")
        local_state_fields(raw, "casting", fields)
        _validate_casting_types(raw["type_map"])
        local_boolean(raw["coerce_on_error"], "coerce_on_error")
        _validate_casting_categories(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local casts, including request-derived legacy pandas vocabularies."""
        if engine not in ("pandas", "polars"):
            return None
        CastingApplier.validate_inference_state(state)
        context = _pandas_cast_context(state) if engine == "pandas" else "row"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the mapping-keyed dual-engine path; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": _casting_apply_polars, "pandas": _casting_apply_pandas}
        )

apply(X, _y, params)

Dispatch through the mapping-keyed dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/casting.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the mapping-keyed dual-engine path; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": _casting_apply_polars, "pandas": _casting_apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local casts, including request-derived legacy pandas vocabularies.

Source code in skyulf-core/skyulf/preprocessing/casting.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local casts, including request-derived legacy pandas vocabularies."""
    if engine not in ("pandas", "polars"):
        return None
    CastingApplier.validate_inference_state(state)
    context = _pandas_cast_context(state) if engine == "pandas" else "row"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect local dtype/vocabulary state without converting or relearning it.

Source code in skyulf-core/skyulf/preprocessing/casting.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect local dtype/vocabulary state without converting or relearning it."""
    fields = {"type", "type_map", "coerce_on_error"}
    if isinstance(raw, dict) and "categories" in raw:
        fields.add("categories")
    local_state_fields(raw, "casting", fields)
    _validate_casting_types(raw["type_map"])
    local_boolean(raw["coerce_on_error"], "coerce_on_error")
    _validate_casting_categories(raw)
    return raw

CastingCalculator

Bases: BaseCalculator

Normalise either accepted config shape into one canonical type_map.

Ordinary casts resolve dtype aliases through TYPE_ALIASES. Category casts additionally learn the observed training vocabulary so applying a held-out batch cannot change the accepted categories or pandas codes.

Source code in skyulf-core/skyulf/preprocessing/casting.py

@NodeRegistry.register("Casting", CastingApplier)
@node_meta(
    id="Casting",
    name="Type Casting",
    category="Data Operations",
    description="Cast columns to specific data types.",
    params={"type_map": {}, "coerce_on_error": True},
    learns_from_data=True,
)
class CastingCalculator(BaseCalculator):
    """Normalise either accepted config shape into one canonical ``type_map``.

    Ordinary casts resolve dtype aliases through ``TYPE_ALIASES``. Category
    casts additionally learn the observed training vocabulary so applying a
    held-out batch cannot change the accepted categories or pandas codes.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Rewrite the dtype labels of ``input_schema`` for every column the config targets.

        The shared ``target_type`` overrides per-column choices, matching fit.
        Columns absent from the input schema are skipped, as they are at runtime.
        """
        # Casting preserves the column set but rewrites dtype labels.
        column_types = dict(config.get("column_types", {}) or {})
        target_type = config.get("target_type")
        columns = config.get("columns", []) or []
        if target_type and columns:
            for col in columns:
                column_types[col] = target_type
        new_schema = input_schema
        for col, dtype in column_types.items():
            resolved = _resolve_casting_dtype(dtype)
            new_schema = new_schema.with_dtype(col, str(resolved))
        return new_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CastingArtifact:  # pylint: disable=arguments-differ
        """Merge the two config shapes into a ``type_map`` of columns present on *X*.

        Absent columns are dropped rather than erroring, so a config written
        against a wider frame degrades to a partial cast; when both shapes name
        the same column, ``target_type`` overwrites the ``column_types`` entry.
        """
        # Config supports two shapes:
        #   {'columns': ['col1'], 'target_type': 'float'}
        #   {'column_types': {'col1': 'float', 'col2': 'int'}}
        target_type = config.get("target_type")
        columns = config.get("columns", [])
        column_types = config.get("column_types", {})

        final_map: dict[str, Any] = {}
        for col, dtype in column_types.items():
            if col in X.columns:
                final_map[col] = _resolve_casting_dtype(dtype)

        if target_type and columns:
            resolved_type = _resolve_casting_dtype(target_type)
            for col in columns:
                if col in X.columns:
                    final_map[col] = resolved_type

        categories = self._fit_categories(X, final_map)

        return {
            "type": "casting",
            "type_map": final_map,
            "coerce_on_error": config.get("coerce_on_error", True),
            "categories": categories,
        }

    @staticmethod
    def _fit_categories(X: Any, type_map: dict[str, Any]) -> dict[str, list[Any]]:
        """Learn observed training categories without retaining unused vocabulary entries."""
        category_columns = [col for col, dtype in type_map.items() if dtype == "category"]
        categories: dict[str, list[Any]] = {}
        if category_columns:
            category_frame = select_then_to_pandas(X, category_columns)
            for col in category_columns:
                # Rebuild from observed values, excluding unused categories
                # that a sliced frame may have inherited from held-out rows.
                categories[col] = pd.Categorical(category_frame[col].to_numpy()).categories.tolist()

        return categories

fit(X, _y, config)

Merge the two config shapes into a type_map of columns present on X.

Absent columns are dropped rather than erroring, so a config written against a wider frame degrades to a partial cast; when both shapes name the same column, target_type overwrites the column_types entry.

Source code in skyulf-core/skyulf/preprocessing/casting.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CastingArtifact:  # pylint: disable=arguments-differ
    """Merge the two config shapes into a ``type_map`` of columns present on *X*.

    Absent columns are dropped rather than erroring, so a config written
    against a wider frame degrades to a partial cast; when both shapes name
    the same column, ``target_type`` overwrites the ``column_types`` entry.
    """
    # Config supports two shapes:
    #   {'columns': ['col1'], 'target_type': 'float'}
    #   {'column_types': {'col1': 'float', 'col2': 'int'}}
    target_type = config.get("target_type")
    columns = config.get("columns", [])
    column_types = config.get("column_types", {})

    final_map: dict[str, Any] = {}
    for col, dtype in column_types.items():
        if col in X.columns:
            final_map[col] = _resolve_casting_dtype(dtype)

    if target_type and columns:
        resolved_type = _resolve_casting_dtype(target_type)
        for col in columns:
            if col in X.columns:
                final_map[col] = resolved_type

    categories = self._fit_categories(X, final_map)

    return {
        "type": "casting",
        "type_map": final_map,
        "coerce_on_error": config.get("coerce_on_error", True),
        "categories": categories,
    }

infer_output_schema(input_schema, config)

Rewrite the dtype labels of input_schema for every column the config targets.

The shared target_type overrides per-column choices, matching fit. Columns absent from the input schema are skipped, as they are at runtime.

Source code in skyulf-core/skyulf/preprocessing/casting.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Rewrite the dtype labels of ``input_schema`` for every column the config targets.

    The shared ``target_type`` overrides per-column choices, matching fit.
    Columns absent from the input schema are skipped, as they are at runtime.
    """
    # Casting preserves the column set but rewrites dtype labels.
    column_types = dict(config.get("column_types", {}) or {})
    target_type = config.get("target_type")
    columns = config.get("columns", []) or []
    if target_type and columns:
        for col in columns:
            column_types[col] = target_type
    new_schema = input_schema
    for col, dtype in column_types.items():
        resolved = _resolve_casting_dtype(dtype)
        new_schema = new_schema.with_dtype(col, str(resolved))
    return new_schema

ClipValuesApplier

Bases: BaseApplier

Set values below lower to lower and above upper to upper.

Nulls stay null, other columns are untouched and every row is kept, so unlike ManualBounds this step also runs on scoring rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

class ClipValuesApplier(BaseApplier):
    """Set values below ``lower`` to ``lower`` and above ``upper`` to ``upper``.

    Nulls stay null, other columns are untouched and every row is kept, so
    unlike ManualBounds this step also runs on scoring rows.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return _clip_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        params = _normalize(raw)
        params.pop("target_column", None)
        params.pop("_auto_columns", None)
        _fields(params, {"bounds"})
        bounds = _bounds(params["bounds"])
        if bounds != state["bounds"]:
            raise ValueError("Configured bounds disagree with fitted clipping bounds.")
        return {"bounds": bounds}

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Clip the configured columns on the active engine; ``y`` passes through."""
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        exprs = [
            _clip_polars_column(X, column, bound)
            for column, bound in params.get("bounds", {}).items()
            if column in X.columns
        ]
        return X.with_columns(exprs), y

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        out = X.copy()
        for column, bound in params.get("bounds", {}).items():
            if column not in out.columns:
                continue
            series = out[column]
            if is_decimal_series(series) or _fractional_bound(bound):
                series = pd.to_numeric(series).astype("float64")
            out[column] = series.clip(lower=bound.get("lower"), upper=bound.get("upper"))
        return out, y

apply(X, y, params)

Clip the configured columns on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Clip the configured columns on the active engine; ``y`` passes through."""
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    params = _normalize(raw)
    params.pop("target_column", None)
    params.pop("_auto_columns", None)
    _fields(params, {"bounds"})
    bounds = _bounds(params["bounds"])
    if bounds != state["bounds"]:
        raise ValueError("Configured bounds disagree with fitted clipping bounds.")
    return {"bounds": bounds}

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return _clip_state(raw)

ClipValuesCalculator

Bases: BaseCalculator

Validate the configured bounds; nothing is learned from the data.

bounds maps a column to {"lower": ..., "upper": ...}; either side may be omitted. Use Winsorize for bounds learned from percentiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

@NodeRegistry.register(
    "ClipValues",
    ClipValuesApplier,
    execution_capabilities=(
        ExecutionCapability("pandas", "apply", "python_batch", "preserve", "row"),
        ExecutionCapability("polars", "apply", "local", "preserve", "row"),
    ),
)
@node_meta(
    id="ClipValues",
    name="Clip Values",
    category="Preprocessing",
    description=(
        "Limits numeric columns to fixed lower/upper bounds (e.g. employees at most 500) "
        "without removing rows."
    ),
    params={"bounds": {}},
    learns_from_data=False,
)
class ClipValuesCalculator(BaseCalculator):
    """Validate the configured bounds; nothing is learned from the data.

    ``bounds`` maps a column to ``{"lower": ..., "upper": ...}``; either side
    may be omitted. Use Winsorize for bounds learned from percentiles.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Promote fractionally bounded columns to match both runtime engines."""
        schema = input_schema
        for column, bound in (config.get("bounds") or {}).items():
            if column in schema.columns and _fractional_bound(bound):
                schema = schema.with_dtype(column, "float64")
        return schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ClipValuesArtifact:  # pylint: disable=arguments-differ
        """Return the validated bounds as the artifact."""
        raw = config.get("bounds") or {}
        bounds = {column: _clean_bound(column, bound) for column, bound in raw.items()}
        if bounds and X is not None:
            _check_columns(X.to_native() if hasattr(X, "to_native") else X, bounds)
        return {"type": "clip_values", "bounds": bounds}

fit(X, _y, config)

Return the validated bounds as the artifact.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ClipValuesArtifact:  # pylint: disable=arguments-differ
    """Return the validated bounds as the artifact."""
    raw = config.get("bounds") or {}
    bounds = {column: _clean_bound(column, bound) for column, bound in raw.items()}
    if bounds and X is not None:
        _check_columns(X.to_native() if hasattr(X, "to_native") else X, bounds)
    return {"type": "clip_values", "bounds": bounds}

infer_output_schema(input_schema, config)

Promote fractionally bounded columns to match both runtime engines.

Source code in skyulf-core/skyulf/preprocessing/outliers/clip_values.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Promote fractionally bounded columns to match both runtime engines."""
    schema = input_schema
    for column, bound in (config.get("bounds") or {}).items():
        if column in schema.columns and _fractional_bound(bound):
            schema = schema.with_dtype(column, "float64")
    return schema

CorrelationThresholdApplier

Bases: BaseApplier

Remove the columns a fitted correlation threshold marked redundant.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

class CorrelationThresholdApplier(BaseApplier):
    """Remove the columns a fitted correlation threshold marked redundant."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved drop decisions without evaluating fit-only metadata.

        Fewer than two fitted candidates produce an empty no-op artifact.
        Threshold and method are unused at apply, including callable methods.
        """
        fields = {"type", "columns_to_drop", "threshold", "method", "drop_columns"}
        if not local_state_fields(raw, "correlation_threshold", fields, allow_empty=True):
            return raw
        _columns(raw["columns_to_drop"])
        _local_selection_drop_flag(raw["drop_columns"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe existing local saved apply without admitting partition workers."""
        if engine not in {"pandas", "polars"}:
            return None
        CorrelationThresholdApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Drop the artifact's precomputed ``columns_to_drop`` from ``X``.

        The list is resolved at fit time, so apply never recomputes a correlation
        matrix; ``drop_columns=False`` leaves the frame untouched.
        """
        return apply_dual_engine(
            X, params, {"polars": _corr_drop_polars, "pandas": _corr_drop_pandas}
        )

apply(X, _y, params)

Drop the artifact's precomputed columns_to_drop from X.

The list is resolved at fit time, so apply never recomputes a correlation matrix; drop_columns=False leaves the frame untouched.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Drop the artifact's precomputed ``columns_to_drop`` from ``X``.

    The list is resolved at fit time, so apply never recomputes a correlation
    matrix; ``drop_columns=False`` leaves the frame untouched.
    """
    return apply_dual_engine(
        X, params, {"polars": _corr_drop_polars, "pandas": _corr_drop_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe existing local saved apply without admitting partition workers.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe existing local saved apply without admitting partition workers."""
    if engine not in {"pandas", "polars"}:
        return None
    CorrelationThresholdApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved drop decisions without evaluating fit-only metadata.

Fewer than two fitted candidates produce an empty no-op artifact. Threshold and method are unused at apply, including callable methods.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved drop decisions without evaluating fit-only metadata.

    Fewer than two fitted candidates produce an empty no-op artifact.
    Threshold and method are unused at apply, including callable methods.
    """
    fields = {"type", "columns_to_drop", "threshold", "method", "drop_columns"}
    if not local_state_fields(raw, "correlation_threshold", fields, allow_empty=True):
        return raw
    _columns(raw["columns_to_drop"])
    _local_selection_drop_flag(raw["drop_columns"])
    return raw

CorrelationThresholdCalculator

Bases: BaseCalculator

Fit a pairwise-correlation threshold over numeric columns.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

@NodeRegistry.register("CorrelationThreshold", CorrelationThresholdApplier)
@node_meta(
    id="CorrelationThreshold",
    name="Correlation Threshold",
    category="Feature Selection",
    description="Remove features highly correlated with others.",
    params={"threshold": 0.95, "method": "pearson"},
    learns_from_data=True,
)
class CorrelationThresholdCalculator(BaseCalculator):
    """Fit a pairwise-correlation threshold over numeric columns."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CorrelationThresholdArtifact:  # pylint: disable=arguments-differ
        """Return columns whose correlation with an earlier one exceeds the threshold.

        Within each over-threshold pair the later column is the one dropped. The
        native Polars path is used only where it can honour the same contract;
        Kendall and callable methods fall back to pandas. Returns an empty
        artifact — a no-op passthrough — when fewer than two columns resolve.
        """
        threshold = config.get("threshold", 0.95)
        drop_columns = config.get("drop_columns", True)
        # Prefer "correlation_method" — falling back to "method" can collide with the
        # facade's own "method" key (e.g. "correlation_threshold").
        method = config.get("correlation_method", "pearson")
        frame = _as_polars_frame(X)

        if frame is not None:
            columns = resolve_columns(SkyulfPolarsWrapper(frame), config, detect_numeric_columns)
            if len(columns) < 2:
                return cast(CorrelationThresholdArtifact, {})
            if _native_polars_correlation_eligible(frame, columns, method, threshold):
                native_method = cast(NativePolarsCorrelationMethod, method)
                return cast(
                    CorrelationThresholdArtifact,
                    {
                        "type": "correlation_threshold",
                        "columns_to_drop": _polars_correlation_columns_to_drop(
                            frame,
                            columns,
                            native_method,
                            threshold,
                        ),
                        "threshold": threshold,
                        "method": method,
                        "drop_columns": drop_columns,
                    },
                )

        # Retain this compatibility route until Polars supports Kendall and callable correlations.
        return _fit_correlation_threshold_pandas(X, config)

fit(X, _y, config)

Return columns whose correlation with an earlier one exceeds the threshold.

Within each over-threshold pair the later column is the one dropped. The native Polars path is used only where it can honour the same contract; Kendall and callable methods fall back to pandas. Returns an empty artifact — a no-op passthrough — when fewer than two columns resolve.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/correlation.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CorrelationThresholdArtifact:  # pylint: disable=arguments-differ
    """Return columns whose correlation with an earlier one exceeds the threshold.

    Within each over-threshold pair the later column is the one dropped. The
    native Polars path is used only where it can honour the same contract;
    Kendall and callable methods fall back to pandas. Returns an empty
    artifact — a no-op passthrough — when fewer than two columns resolve.
    """
    threshold = config.get("threshold", 0.95)
    drop_columns = config.get("drop_columns", True)
    # Prefer "correlation_method" — falling back to "method" can collide with the
    # facade's own "method" key (e.g. "correlation_threshold").
    method = config.get("correlation_method", "pearson")
    frame = _as_polars_frame(X)

    if frame is not None:
        columns = resolve_columns(SkyulfPolarsWrapper(frame), config, detect_numeric_columns)
        if len(columns) < 2:
            return cast(CorrelationThresholdArtifact, {})
        if _native_polars_correlation_eligible(frame, columns, method, threshold):
            native_method = cast(NativePolarsCorrelationMethod, method)
            return cast(
                CorrelationThresholdArtifact,
                {
                    "type": "correlation_threshold",
                    "columns_to_drop": _polars_correlation_columns_to_drop(
                        frame,
                        columns,
                        native_method,
                        threshold,
                    ),
                    "threshold": threshold,
                    "method": method,
                    "drop_columns": drop_columns,
                },
            )

    # Retain this compatibility route until Polars supports Kendall and callable correlations.
    return _fit_correlation_threshold_pandas(X, config)

CountVectorizerApplier

Bases: BaseApplier

Attach one token-count column per vocabulary term to the frame.

The configured text columns are joined into a single corpus string before transforming, the dense result is concatenated on, and the source columns survive unless drop_original is set. This node uses the text-specific dispatcher: the polars path runs natively, so only the text payload crosses into sklearn, and it falls back to a full pandas round-trip when a text column is not String dtype.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

class CountVectorizerApplier(BaseApplier):
    """Attach one token-count column per vocabulary term to the frame.

    The configured text columns are joined into a single corpus string before
    transforming, the dense result is concatenated on, and the source columns
    survive unless ``drop_original`` is set. This node uses the text-specific
    dispatcher: the polars path runs natively, so only the text payload crosses
    into sklearn, and it falls back to a full pandas round-trip when a text
    column is not String dtype.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved vocabulary and binary settings without fitting or transforming."""
        vectorizer = _validate_local_vectorizer(
            raw, "count_vectorizer", CountVectorizer, {"vocabulary", "max_features", "binary"}
        )
        if vectorizer is not None:
            _validate_vectorizer_vocabulary(raw, vectorizer)
            local_boolean(raw["binary"], "binary")
            if raw["binary"] != vectorizer.binary:
                raise ValueError("Fitted binary setting disagrees with the saved vectorizer.")
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe fixed per-document counts without promising callback or memory safety."""
        if engine not in ("pandas", "polars"):
            return None
        CountVectorizerApplier.validate_inference_state(state)
        if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
        return apply_text_dual_engine(
            X, params, _count_apply_pandas, _sklearn_vectorizer_apply_polars
        )

apply(X, _y, params)

Dispatch through the text-specific dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
    return apply_text_dual_engine(
        X, params, _count_apply_pandas, _sklearn_vectorizer_apply_polars
    )

inference_capability(state, *, engine) staticmethod

Describe fixed per-document counts without promising callback or memory safety.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe fixed per-document counts without promising callback or memory safety."""
    if engine not in ("pandas", "polars"):
        return None
    CountVectorizerApplier.validate_inference_state(state)
    if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved vocabulary and binary settings without fitting or transforming.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved vocabulary and binary settings without fitting or transforming."""
    vectorizer = _validate_local_vectorizer(
        raw, "count_vectorizer", CountVectorizer, {"vocabulary", "max_features", "binary"}
    )
    if vectorizer is not None:
        _validate_vectorizer_vocabulary(raw, vectorizer)
        local_boolean(raw["binary"], "binary")
        if raw["binary"] != vectorizer.binary:
            raise ValueError("Fitted binary setting disagrees with the saved vectorizer.")
    return raw

CountVectorizerCalculator

Bases: BaseCalculator

Fit a CountVectorizer vocabulary on the joined text of the resolved columns.

The vocabulary — and therefore the output width — is learned from the training corpus, so the artifact is not derivable from config alone. A vocabulary over _LARGE_VOCAB_THRESHOLD columns is warned about rather than raising, since a wide bag-of-words is a memory problem, not an error.

Schema inference inherits the base None result: the learned vocabulary determines the output columns, so their schema is only known at runtime.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

@NodeRegistry.register("count_vectorizer", CountVectorizerApplier)
@node_meta(
    id="count_vectorizer",
    name="Count Vectorizer",
    category="Text",
    description=(
        "Convert text columns to token-count feature columns (bag-of-words). "
        "Fits a vocabulary on training data; apply uses that vocabulary."
    ),
    params={
        "columns": [],
        "max_features": None,
        "min_df": 1,
        "max_df": 1.0,
        "ngram_range": [1, 1],
        "lowercase": True,
        "stop_words": None,
        "binary": False,
        "drop_original": False,
    },
    tags=["text", "nlp", "vectorizer", "bag-of-words"],
    learns_from_data=True,
)
class CountVectorizerCalculator(BaseCalculator):
    """Fit a ``CountVectorizer`` vocabulary on the joined text of the resolved columns.

    The vocabulary — and therefore the output width — is learned from the
    training corpus, so the artifact is not derivable from config alone. A
    vocabulary over ``_LARGE_VOCAB_THRESHOLD`` columns is warned about rather
    than raising, since a wide bag-of-words is a memory problem, not an error.

    Schema inference inherits the base ``None`` result: the learned vocabulary
    determines the output columns, so their schema is only known at runtime.
    """

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CountVectorizerArtifact:  # pylint: disable=arguments-differ
        """Narrow to the configured text columns and fit the vocabulary on their joined text.

        An empty or wholly-absent column selection yields an empty artifact so
        the applier no-ops. An empty vocabulary also yields an empty artifact
        with a warning; source columns are retained even with ``drop_original``.
        """
        resolved = resolve_fit_text_columns(X, config, _y)
        if resolved is None:
            return {}
        text_frame, valid_cols = resolved

        artifact = _build_count_artifact(config, text_frame, valid_cols)
        validate_text_output_names(X, artifact, valid_cols)
        return artifact

fit(X, _y, config)

Narrow to the configured text columns and fit the vocabulary on their joined text.

An empty or wholly-absent column selection yields an empty artifact so the applier no-ops. An empty vocabulary also yields an empty artifact with a warning; source columns are retained even with drop_original.

Source code in skyulf-core/skyulf/preprocessing/vectorization/count_vectorizer.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> CountVectorizerArtifact:  # pylint: disable=arguments-differ
    """Narrow to the configured text columns and fit the vocabulary on their joined text.

    An empty or wholly-absent column selection yields an empty artifact so
    the applier no-ops. An empty vocabulary also yields an empty artifact
    with a warning; source columns are retained even with ``drop_original``.
    """
    resolved = resolve_fit_text_columns(X, config, _y)
    if resolved is None:
        return {}
    text_frame, valid_cols = resolved

    artifact = _build_count_artifact(config, text_frame, valid_cols)
    validate_text_output_names(X, artifact, valid_cols)
    return artifact

CustomBinningApplier

Bases: GeneralBinningApplier

Describe fixed-edge inference while reusing the shared binning apply logic.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

class CustomBinningApplier(GeneralBinningApplier):
    """Describe fixed-edge inference while reusing the shared binning apply logic."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect actual local fitted state unchanged, including the explicit empty no-op."""
        return _validate_binning_inference_state(raw, allow_custom_labels=False)

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe row dependencies; exact batch schema equivalence needs a real probe."""
        if engine not in ("pandas", "polars"):
            return None
        CustomBinningApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

inference_capability(state, *, engine) staticmethod

Describe row dependencies; exact batch schema equivalence needs a real probe.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe row dependencies; exact batch schema equivalence needs a real probe."""
    if engine not in ("pandas", "polars"):
        return None
    CustomBinningApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect actual local fitted state unchanged, including the explicit empty no-op.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect actual local fitted state unchanged, including the explicit empty no-op."""
    return _validate_binning_inference_state(raw, allow_custom_labels=False)

CustomBinningCalculator

Bases: BaseCalculator

Apply user-supplied bin edges to selected columns.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@NodeRegistry.register("CustomBinning", CustomBinningApplier)
@node_meta(
    id="CustomBinning",
    name="Custom Binning",
    category="Preprocessing",
    description="Bin data using custom edges.",
    params={"bins": [], "columns": []},
    learns_from_data=True,
)
class CustomBinningCalculator(BaseCalculator):
    """Apply user-supplied bin edges to selected columns."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralBinningArtifact:
        """Attach one shared, sorted edge list from ``config["bins"]`` to every selected column.

        Edges are fixed, but automatic column selection examines observed
        values to exclude binary, constant, and all-missing columns.
        Unlike :class:`GeneralBinningCalculator`
        the same edges go to every column, so a multi-column selection spanning
        different scales will push the out-of-scale columns' values outside the
        outer bins, where both engines null them. An empty or absent ``bins``
        leaves ``bin_edges`` empty, which makes apply a no-op.
        """
        if user_picked_no_columns(config):
            return cast(GeneralBinningArtifact, {})

        input_columns = list(X.columns)
        X, columns = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        bins = config.get("bins")

        bin_edges_map: dict[str, list[float]] = {}
        if bins:
            sorted_bins = sorted(bins)
            for col in columns:
                if col in X.columns:
                    bin_edges_map[col] = sorted_bins

        artifact: dict[str, Any] = {
            "type": "general_binning",  # Reuses GeneralBinningApplier.
            "bin_edges": bin_edges_map,
        }
        artifact.update(_passthrough_artifact_options(config))
        _validate_binning_artifact(input_columns, artifact)
        _store_range_labels(artifact)
        return cast(GeneralBinningArtifact, artifact)

fit(X, _y, config)

Attach one shared, sorted edge list from config["bins"] to every selected column.

Edges are fixed, but automatic column selection examines observed values to exclude binary, constant, and all-missing columns. Unlike :class:GeneralBinningCalculator the same edges go to every column, so a multi-column selection spanning different scales will push the out-of-scale columns' values outside the outer bins, where both engines null them. An empty or absent bins leaves bin_edges empty, which makes apply a no-op.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralBinningArtifact:
    """Attach one shared, sorted edge list from ``config["bins"]`` to every selected column.

    Edges are fixed, but automatic column selection examines observed
    values to exclude binary, constant, and all-missing columns.
    Unlike :class:`GeneralBinningCalculator`
    the same edges go to every column, so a multi-column selection spanning
    different scales will push the out-of-scale columns' values outside the
    outer bins, where both engines null them. An empty or absent ``bins``
    leaves ``bin_edges`` empty, which makes apply a no-op.
    """
    if user_picked_no_columns(config):
        return cast(GeneralBinningArtifact, {})

    input_columns = list(X.columns)
    X, columns = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    bins = config.get("bins")

    bin_edges_map: dict[str, list[float]] = {}
    if bins:
        sorted_bins = sorted(bins)
        for col in columns:
            if col in X.columns:
                bin_edges_map[col] = sorted_bins

    artifact: dict[str, Any] = {
        "type": "general_binning",  # Reuses GeneralBinningApplier.
        "bin_edges": bin_edges_map,
    }
    artifact.update(_passthrough_artifact_options(config))
    _validate_binning_artifact(input_columns, artifact)
    _store_range_labels(artifact)
    return cast(GeneralBinningArtifact, artifact)

DataSnapshotApplier

Bases: BaseApplier

Passthrough applier; the snapshot is produced entirely at fit time.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

class DataSnapshotApplier(BaseApplier):
    """Passthrough applier; the snapshot is produced entirely at fit time."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Check ownership without interpreting or recapturing training values."""
        local_state_fields(raw, "data_snapshot", {"type", "snapshot"})
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe the existing passthrough without taking another snapshot."""
        if engine not in {"pandas", "polars"}:
            return None
        DataSnapshotApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    def apply(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        params: dict[str, Any],
    ) -> Any:
        """Return ``df`` unchanged, ignoring the captured snapshot."""
        return df

apply(df, params)

Return df unchanged, ignoring the captured snapshot.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

def apply(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    params: dict[str, Any],
) -> Any:
    """Return ``df`` unchanged, ignoring the captured snapshot."""
    return df

inference_capability(state, *, engine) staticmethod

Describe the existing passthrough without taking another snapshot.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe the existing passthrough without taking another snapshot."""
    if engine not in {"pandas", "polars"}:
        return None
    DataSnapshotApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Check ownership without interpreting or recapturing training values.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Check ownership without interpreting or recapturing training values."""
    local_state_fields(raw, "data_snapshot", {"type", "snapshot"})
    return raw

DataSnapshotCalculator

Bases: BaseCalculator

Capture the leading rows of the input as plain record dicts.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@NodeRegistry.register("DataSnapshot", DataSnapshotApplier)
@node_meta(
    id="DataSnapshot",
    name="Data Snapshot",
    category="Inspection",
    description="Take a snapshot of the first N rows of the dataset.",
    params={"n_rows": 5},
    learns_from_data=False,
)
class DataSnapshotCalculator(BaseCalculator):
    """Capture the leading rows of the input as plain record dicts."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Inspection nodes are read-only; schema is unchanged.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DataSnapshotArtifact:  # pylint: disable=arguments-differ
        """Return the first ``config["n_rows"]`` rows (default 5) as dicts.

        Rows are serialised eagerly at fit time, so the snapshot is unaffected
        by later mutation of the source frame.
        """
        return cast(
            DataSnapshotArtifact,
            fit_dual_engine(
                X, config, {"polars": _snapshot_fit_polars, "pandas": _snapshot_fit_pandas}
            ),
        )

fit(X, _y, config)

Return the first config["n_rows"] rows (default 5) as dicts.

Rows are serialised eagerly at fit time, so the snapshot is unaffected by later mutation of the source frame.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DataSnapshotArtifact:  # pylint: disable=arguments-differ
    """Return the first ``config["n_rows"]`` rows (default 5) as dicts.

    Rows are serialised eagerly at fit time, so the snapshot is unaffected
    by later mutation of the source frame.
    """
    return cast(
        DataSnapshotArtifact,
        fit_dual_engine(
            X, config, {"polars": _snapshot_fit_polars, "pandas": _snapshot_fit_pandas}
        ),
    )

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Inspection nodes are read-only; schema is unchanged.
    return input_schema

DatasetProfileApplier

Bases: BaseApplier

Passthrough applier; the profile is produced entirely at fit time.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

class DatasetProfileApplier(BaseApplier):
    """Passthrough applier; the profile is produced entirely at fit time."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Check ownership while leaving the unused training report untouched."""
        local_state_fields(raw, "dataset_profile", {"type", "profile"})
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe the existing passthrough without collecting new statistics."""
        if engine not in {"pandas", "polars"}:
            return None
        DatasetProfileApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    def apply(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        params: dict[str, Any],
    ) -> Any:
        """Return ``df`` unchanged, ignoring the fitted profile."""
        # Inspection nodes do not modify data.
        return df

apply(df, params)

Return df unchanged, ignoring the fitted profile.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

def apply(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    params: dict[str, Any],
) -> Any:
    """Return ``df`` unchanged, ignoring the fitted profile."""
    # Inspection nodes do not modify data.
    return df

inference_capability(state, *, engine) staticmethod

Describe the existing passthrough without collecting new statistics.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe the existing passthrough without collecting new statistics."""
    if engine not in {"pandas", "polars"}:
        return None
    DatasetProfileApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Check ownership while leaving the unused training report untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Check ownership while leaving the unused training report untouched."""
    local_state_fields(raw, "dataset_profile", {"type", "profile"})
    return raw

DatasetProfileCalculator

Bases: BaseCalculator

Summarise shape, dtypes, missingness and numeric statistics.

Numeric statistics include supported integer, unsigned integer, float and Decimal columns, even when binary, constant, entirely missing or empty. Boolean, categorical and temporal columns retain their dtype/missingness metadata but are not treated as numeric features.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@NodeRegistry.register("DatasetProfile", DatasetProfileApplier)
@node_meta(
    id="DatasetProfile",
    name="Dataset Profile",
    category="Inspection",
    description="Generate a statistical profile of the dataset.",
    params={},
    learns_from_data=False,
)
class DatasetProfileCalculator(BaseCalculator):
    """Summarise shape, dtypes, missingness and numeric statistics.

    Numeric statistics include supported integer, unsigned integer, float and
    Decimal columns, even when binary, constant, entirely missing or empty.
    Boolean, categorical and temporal columns retain their dtype/missingness
    metadata but are not treated as numeric features.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Inspection nodes are read-only; schema is unchanged.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DatasetProfileArtifact:  # pylint: disable=arguments-differ
        """Collect row/column counts, dtypes, missing counts and numeric stats.

        Numeric statistics come from Polars ``describe()`` or pandas
        ``describe()`` depending on the engine, so the per-column metric names
        are not identical across engines.
        """
        return cast(
            DatasetProfileArtifact,
            fit_dual_engine(
                X, config, {"polars": _profile_fit_polars, "pandas": _profile_fit_pandas}
            ),
        )

fit(X, _y, config)

Collect row/column counts, dtypes, missing counts and numeric stats.

Numeric statistics come from Polars describe() or pandas describe() depending on the engine, so the per-column metric names are not identical across engines.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DatasetProfileArtifact:  # pylint: disable=arguments-differ
    """Collect row/column counts, dtypes, missing counts and numeric stats.

    Numeric statistics come from Polars ``describe()`` or pandas
    ``describe()`` depending on the engine, so the per-column metric names
    are not identical across engines.
    """
    return cast(
        DatasetProfileArtifact,
        fit_dual_engine(
            X, config, {"polars": _profile_fit_polars, "pandas": _profile_fit_pandas}
        ),
    )

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Inspection nodes are read-only; schema is unchanged.
    return input_schema

DateFeaturesApplier

Bases: BaseApplier

Append calendar-part columns extracted from the configured datetime columns.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

class DateFeaturesApplier(BaseApplier):
    """Append calendar-part columns extracted from the configured datetime columns."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved calendar settings without parsing data or changing legacy defaults."""
        optional = {"drop_original", "timezone", "epoch_unit"}
        fields = {"type", "columns", "features"}
        if type(raw) is dict:
            fields.update(optional.intersection(raw))
        local_state_fields(raw, "date_features", fields)
        _validate_date_names(raw["columns"])
        _validate_date_names(raw["features"])
        if any(feature not in DATE_FEATURE_ACCESSORS for feature in raw["features"]):
            raise ValueError("Fitted date features contain unsupported calendar parts.")
        local_boolean(raw.get("drop_original", False), "drop_original")
        if raw.get("timezone") is not None and type(raw["timezone"]) is not str:
            raise ValueError("Fitted date timezone must be a string or None.")
        _epoch_unit(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved UTC parsing without promising legacy mixed-timezone behavior."""
        if engine not in ("pandas", "polars"):
            return None
        DateFeaturesApplier.validate_inference_state(state)
        if state.get("timezone") != "UTC":
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Extract the configured calendar features on the active engine; ``y`` passes through."""
        return apply_dual_engine(X, params, {"polars": _apply_polars, "pandas": _apply_pandas})

apply(X, _y, params)

Extract the configured calendar features on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Extract the configured calendar features on the active engine; ``y`` passes through."""
    return apply_dual_engine(X, params, {"polars": _apply_polars, "pandas": _apply_pandas})

inference_capability(state, *, engine) staticmethod

Describe saved UTC parsing without promising legacy mixed-timezone behavior.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved UTC parsing without promising legacy mixed-timezone behavior."""
    if engine not in ("pandas", "polars"):
        return None
    DateFeaturesApplier.validate_inference_state(state)
    if state.get("timezone") != "UTC":
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved calendar settings without parsing data or changing legacy defaults.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved calendar settings without parsing data or changing legacy defaults."""
    optional = {"drop_original", "timezone", "epoch_unit"}
    fields = {"type", "columns", "features"}
    if type(raw) is dict:
        fields.update(optional.intersection(raw))
    local_state_fields(raw, "date_features", fields)
    _validate_date_names(raw["columns"])
    _validate_date_names(raw["features"])
    if any(feature not in DATE_FEATURE_ACCESSORS for feature in raw["features"]):
        raise ValueError("Fitted date features contain unsupported calendar parts.")
    local_boolean(raw.get("drop_original", False), "drop_original")
    if raw.get("timezone") is not None and type(raw["timezone"]) is not str:
        raise ValueError("Fitted date timezone must be a string or None.")
    _epoch_unit(raw)
    return raw

DateFeaturesCalculator

Bases: BaseCalculator

Record calendar features using UTC for every newly fitted artifact.

Offset-aware timestamps are converted to UTC before extracting their calendar parts; naive values are treated as UTC without shifting their clock time. Unparseable values produce null features. Artifacts fitted before this contract retain their original engine-specific interpretation so an existing model's input features do not silently change on reload. Refit the complete pipeline to adopt UTC for such models. Numeric epochs require an explicit epoch_unit (s, ms, us or ns), including on replay of legacy artifacts; units are never inferred from numeric magnitude.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

@NodeRegistry.register("DateFeatures", DateFeaturesApplier)
@node_meta(
    id="DateFeatures",
    name="Date Features",
    category="Preprocessing",
    description="Extract calendar parts (year, month, day-of-week, ...) from datetime columns.",
    params={
        "columns": [],
        "features": DEFAULT_FEATURES,
        "drop_original": False,
        "epoch_unit": None,
    },
    tags=["time-series"],
    learns_from_data=False,
)
class DateFeaturesCalculator(BaseCalculator):
    """Record calendar features using UTC for every newly fitted artifact.

    Offset-aware timestamps are converted to UTC before extracting their
    calendar parts; naive values are treated as UTC without shifting their
    clock time. Unparseable values produce null features. Artifacts fitted
    before this contract retain their original engine-specific interpretation
    so an existing model's input features do not silently change on reload.
    Refit the complete pipeline to adopt UTC for such models. Numeric epochs
    require an explicit ``epoch_unit`` (s, ms, us or ns), including on replay
    of legacy artifacts; units are never inferred from numeric magnitude.
    """

    def fit(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        config: dict[str, Any],
    ) -> DateFeaturesArtifact:
        """Record the columns and supported features, defaulting to year/month/day/dayofweek."""
        unit = _epoch_unit(config)
        _validate_numeric_date_columns(df, config.get("columns", []), unit)
        return {
            "type": "date_features",
            "columns": config.get("columns", []),
            "features": _resolve_features(config),
            "drop_original": bool(config.get("drop_original", False)),
            "timezone": "UTC",
            "epoch_unit": unit,
        }

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Add a nullable-int column per (column, feature); drop originals on request."""
        # Calendar parts are nullable integers (an unparseable date yields a
        # null feature value, not 0/NaN) - "Int64" here is a best-effort,
        # engine-agnostic label communicating nullability; the *actual*
        # column width still varies per engine/feature (e.g. polars uses
        # Int8/Int16/Int32 for narrower fields like month/day/hour), so this
        # is intentionally not asserted via check_dtypes=True comparisons.
        cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
        features = _resolve_features(config)
        schema = input_schema
        for col in cols:
            for feature in features:
                schema = schema.add(_feat_name(col, feature), "Int64")
        if config.get("drop_original"):
            schema = schema.drop(cols)
        return schema

fit(df, config)

Record the columns and supported features, defaulting to year/month/day/dayofweek.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

def fit(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    config: dict[str, Any],
) -> DateFeaturesArtifact:
    """Record the columns and supported features, defaulting to year/month/day/dayofweek."""
    unit = _epoch_unit(config)
    _validate_numeric_date_columns(df, config.get("columns", []), unit)
    return {
        "type": "date_features",
        "columns": config.get("columns", []),
        "features": _resolve_features(config),
        "drop_original": bool(config.get("drop_original", False)),
        "timezone": "UTC",
        "epoch_unit": unit,
    }

infer_output_schema(input_schema, config)

Add a nullable-int column per (column, feature); drop originals on request.

Source code in skyulf-core/skyulf/preprocessing/time_series/date_features.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Add a nullable-int column per (column, feature); drop originals on request."""
    # Calendar parts are nullable integers (an unparseable date yields a
    # null feature value, not 0/NaN) - "Int64" here is a best-effort,
    # engine-agnostic label communicating nullability; the *actual*
    # column width still varies per engine/feature (e.g. polars uses
    # Int8/Int16/Int32 for narrower fields like month/day/hour), so this
    # is intentionally not asserted via check_dtypes=True comparisons.
    cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
    features = _resolve_features(config)
    schema = input_schema
    for col in cols:
        for feature in features:
            schema = schema.add(_feat_name(col, feature), "Int64")
    if config.get("drop_original"):
        schema = schema.drop(cols)
    return schema

DeduplicateApplier

Bases: BaseApplier

Drop duplicate rows, keeping y aligned with the survivors.

Both engines must agree on which rows survive. y is filtered by kept positions, never by label — a label-based .loc returns every row matching a duplicated index label. On polars the dedup key has to be resolved to the real columns before the tracking row index is added, because unique(subset=None) would include that always-unique index and defeat deduplication entirely. keep="none" drops every member of a duplicate group and is spelled False for pandas but "none" for polars.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

class DeduplicateApplier(BaseApplier):
    """Drop duplicate rows, keeping ``y`` aligned with the survivors.

    Both engines must agree on which rows survive. ``y`` is filtered by kept
    *positions*, never by label — a label-based ``.loc`` returns every row
    matching a duplicated index label. On polars the dedup key has to be
    resolved to the real columns before the tracking row index is added,
    because ``unique(subset=None)`` would include that always-unique index and
    defeat deduplication entirely. ``keep="none"`` drops every member of a
    duplicate group and is spelled ``False`` for pandas but ``"none"`` for
    polars.
    """

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Require complete duplicate groups before choosing which rows survive."""
        if engine not in ("pandas", "polars"):
            return None
        return ExecutionCapability(engine, "apply", "local", "filter", "global")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific dedup, forwarding ``(X, y)`` only when ``y`` exists."""
        # Note: dedup must propagate row drops to y, so we route X+y as a tuple
        # through apply_dual_engine which handles unpack/pack.
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"polars": _dedup_apply_polars, "pandas": _dedup_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific dedup, forwarding (X, y) only when y exists.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific dedup, forwarding ``(X, y)`` only when ``y`` exists."""
    # Note: dedup must propagate row drops to y, so we route X+y as a tuple
    # through apply_dual_engine which handles unpack/pack.
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"polars": _dedup_apply_polars, "pandas": _dedup_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Require complete duplicate groups before choosing which rows survive.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Require complete duplicate groups before choosing which rows survive."""
    if engine not in ("pandas", "polars"):
        return None
    return ExecutionCapability(engine, "apply", "local", "filter", "global")

DeduplicateCalculator

Bases: BaseCalculator

Resolve dedup config into an artifact.

The node is flagged learns_from_data because which rows survive depends on the data, but fit itself only reads config.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

@NodeRegistry.register("Deduplicate", DeduplicateApplier)
@node_meta(
    id="Deduplicate",
    name="Deduplicate",
    category="Data Operations",
    description="Drop duplicate rows.",
    params={"subset": [], "keep": "first"},
    learns_from_data=True,
)
class DeduplicateCalculator(BaseCalculator):
    """Resolve dedup config into an artifact.

    The node is flagged ``learns_from_data`` because which rows survive depends
    on the data, but ``fit`` itself only reads ``config``.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Pass the input schema through: rows are dropped, columns are preserved."""
        # Deduplication removes rows; column set is preserved.
        return input_schema

    def fit(self, df: Any, config: dict[str, Any]) -> DeduplicateArtifact:
        """Carry the ``subset`` and ``keep`` policy into the artifact."""
        return {
            "type": "deduplicate",
            "subset": config.get("subset"),
            "keep": config.get("keep", "first"),
        }

fit(df, config)

Carry the subset and keep policy into the artifact.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

def fit(self, df: Any, config: dict[str, Any]) -> DeduplicateArtifact:
    """Carry the ``subset`` and ``keep`` policy into the artifact."""
    return {
        "type": "deduplicate",
        "subset": config.get("subset"),
        "keep": config.get("keep", "first"),
    }

infer_output_schema(input_schema, config)

Pass the input schema through: rows are dropped, columns are preserved.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/deduplicate.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Pass the input schema through: rows are dropped, columns are preserved."""
    # Deduplication removes rows; column set is preserved.
    return input_schema

DropMissingColumnsApplier

Bases: BaseApplier

Drop the columns the calculator selected, skipping names no longer on the frame.

The selection is already baked into params; this half is a pure drop and leaves y alone, so both engine paths are identical by construction.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

class DropMissingColumnsApplier(BaseApplier):
    """Drop the columns the calculator selected, skipping names no longer on the frame.

    The selection is already baked into ``params``; this half is a pure drop
    and leaves ``y`` alone, so both engine paths are identical by construction.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved drop set; threshold metadata no longer controls apply."""
        local_state_fields(raw, "drop_missing_columns", {"type", "columns_to_drop", "threshold"})
        _columns(raw["columns_to_drop"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local saved apply context without granting worker execution."""
        if engine not in ("pandas", "polars"):
            return None
        DropMissingColumnsApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch the column drop to the pandas or polars path; ``y`` passes through."""
        return apply_dual_engine(
            X,
            params,
            {"polars": _drop_missing_cols_apply_polars, "pandas": _drop_missing_cols_apply_pandas},
        )

apply(X, _y, params)

Dispatch the column drop to the pandas or polars path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch the column drop to the pandas or polars path; ``y`` passes through."""
    return apply_dual_engine(
        X,
        params,
        {"polars": _drop_missing_cols_apply_polars, "pandas": _drop_missing_cols_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe local saved apply context without granting worker execution.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local saved apply context without granting worker execution."""
    if engine not in ("pandas", "polars"):
        return None
    DropMissingColumnsApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect the saved drop set; threshold metadata no longer controls apply.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved drop set; threshold metadata no longer controls apply."""
    local_state_fields(raw, "drop_missing_columns", {"type", "columns_to_drop", "threshold"})
    _columns(raw["columns_to_drop"])
    return raw

DropMissingColumnsCalculator

Bases: BaseCalculator

Select columns to drop: an explicit list unioned with those over the missing-% threshold.

The threshold half is data-dependent, so fit runs an engine-specific path and the two must agree on the missing count. They do not for free: polars' null_count() ignores NaN where pandas' isna() counts it, so float columns add an explicit NaN count on the polars side.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

@NodeRegistry.register("DropMissingColumns", DropMissingColumnsApplier)
@node_meta(
    id="DropMissingColumns",
    name="Drop Missing Columns",
    category="Cleaning",
    description="Drop columns that exceed missing value threshold.",
    params={"missing_threshold": 50},
    learns_from_data=True,
)
class DropMissingColumnsCalculator(BaseCalculator):
    """Select columns to drop: an explicit list unioned with those over the missing-% threshold.

    The threshold half is data-dependent, so ``fit`` runs an engine-specific
    path and the two must agree on the missing count. They do not for free:
    polars' ``null_count()`` ignores NaN where pandas' ``isna()`` counts it, so
    float columns add an explicit NaN count on the polars side.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Predict the surviving columns, or ``None`` when a threshold makes that data-dependent.

        Without a positive threshold the result is exactly the input minus the
        explicit list; with one, which columns go depends on the data and
        callers must fall back to runtime introspection.
        """
        # Threshold path is data-dependent; only predictable when the user
        # supplied an explicit column list and no positive threshold.
        if _resolve_threshold(config.get("missing_threshold")) is not None:
            return None
        explicit = config.get("columns", []) or []
        return input_schema if not explicit else input_schema.drop(explicit)

    def fit(self, df: Any, config: dict[str, Any]) -> DropMissingColumnsArtifact:
        """Compute the drop set on the frame's own engine and return it as the artifact."""
        return cast(
            DropMissingColumnsArtifact,
            fit_dual_engine(
                df,
                config,
                {"polars": _drop_missing_cols_fit_polars, "pandas": _drop_missing_cols_fit_pandas},
            ),
        )

fit(df, config)

Compute the drop set on the frame's own engine and return it as the artifact.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

def fit(self, df: Any, config: dict[str, Any]) -> DropMissingColumnsArtifact:
    """Compute the drop set on the frame's own engine and return it as the artifact."""
    return cast(
        DropMissingColumnsArtifact,
        fit_dual_engine(
            df,
            config,
            {"polars": _drop_missing_cols_fit_polars, "pandas": _drop_missing_cols_fit_pandas},
        ),
    )

infer_output_schema(input_schema, config)

Predict the surviving columns, or None when a threshold makes that data-dependent.

Without a positive threshold the result is exactly the input minus the explicit list; with one, which columns go depends on the data and callers must fall back to runtime introspection.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_columns.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Predict the surviving columns, or ``None`` when a threshold makes that data-dependent.

    Without a positive threshold the result is exactly the input minus the
    explicit list; with one, which columns go depends on the data and
    callers must fall back to runtime introspection.
    """
    # Threshold path is data-dependent; only predictable when the user
    # supplied an explicit column list and no positive threshold.
    if _resolve_threshold(config.get("missing_threshold")) is not None:
        return None
    explicit = config.get("columns", []) or []
    return input_schema if not explicit else input_schema.drop(explicit)

DropMissingRowsApplier

Bases: BaseApplier

Drop rows exceeding the configured missingness policy, keeping y in sync.

This node changes the row count, so the pandas and polars paths must drop the same rows and filter y by the identical kept positions. The pandas path therefore computes its keep mask positionally: label-based .loc returns every row matching a duplicated index label, which desynchronises y from the cleaned X. A subset that is empty or names only nonexistent columns collapses to None, which both engines read as "check every column" rather than as a no-op.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

class DropMissingRowsApplier(BaseApplier):
    """Drop rows exceeding the configured missingness policy, keeping ``y`` in sync.

    This node changes the row count, so the pandas and polars paths must drop
    the same rows *and* filter ``y`` by the identical kept positions. The
    pandas path therefore computes its keep mask positionally: label-based
    ``.loc`` returns every row matching a duplicated index label, which
    desynchronises ``y`` from the cleaned ``X``. A ``subset`` that is empty or
    names only nonexistent columns collapses to ``None``, which both engines
    read as "check every column" rather than as a no-op.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved policy without filtering data or converting native values."""
        local_state_fields(
            raw, "drop_missing_rows", {"type", "subset", "how", "threshold", "missing_threshold"}
        )
        _validate_how(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe native filtering; ordinary prediction skips this training policy."""
        if engine not in ("pandas", "polars"):
            return None
        DropMissingRowsApplier.validate_inference_state(state)
        subset = state["subset"]
        if subset is not None and not isinstance(subset, (str, list, tuple, dict, set, frozenset)):
            return None
        threshold = state["threshold"]
        if threshold is None:
            threshold = state["missing_threshold"]
        if threshold is not None and not isinstance(threshold, (Number, np.bool_)):
            # Array/Series thresholds can depend on request positions or indexes.
            return None
        return ExecutionCapability(engine, "apply", "local", "filter", "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific dropna, forwarding ``(X, y)`` only when ``y`` exists."""
        _validate_how(params)
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"polars": _drop_missing_rows_apply_polars, "pandas": _drop_missing_rows_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific dropna, forwarding (X, y) only when y exists.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific dropna, forwarding ``(X, y)`` only when ``y`` exists."""
    _validate_how(params)
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"polars": _drop_missing_rows_apply_polars, "pandas": _drop_missing_rows_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe native filtering; ordinary prediction skips this training policy.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe native filtering; ordinary prediction skips this training policy."""
    if engine not in ("pandas", "polars"):
        return None
    DropMissingRowsApplier.validate_inference_state(state)
    subset = state["subset"]
    if subset is not None and not isinstance(subset, (str, list, tuple, dict, set, frozenset)):
        return None
    threshold = state["threshold"]
    if threshold is None:
        threshold = state["missing_threshold"]
    if threshold is not None and not isinstance(threshold, (Number, np.bool_)):
        # Array/Series thresholds can depend on request positions or indexes.
        return None
    return ExecutionCapability(engine, "apply", "local", "filter", "row")

validate_inference_state(raw) staticmethod

Inspect the saved policy without filtering data or converting native values.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved policy without filtering data or converting native values."""
    local_state_fields(
        raw, "drop_missing_rows", {"type", "subset", "how", "threshold", "missing_threshold"}
    )
    _validate_how(raw)
    return raw

DropMissingRowsCalculator

Bases: BaseCalculator

Resolve drop-missing-rows config into an artifact; nothing is learned from data.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

@NodeRegistry.register("DropMissingRows", DropMissingRowsApplier)
@node_meta(
    id="DropMissingRows",
    name="Drop Missing Rows",
    category="Cleaning",
    description="Drop rows containing missing values in specified columns.",
    params={"subset": [], "how": "any", "missing_threshold": None},
    learns_from_data=False,
)
class DropMissingRowsCalculator(BaseCalculator):
    """Resolve drop-missing-rows config into an artifact; nothing is learned from data."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Pass the input schema through: rows are dropped, columns are preserved."""
        # Drops rows; column set is preserved.
        return input_schema

    def fit(self, df: Any, config: dict[str, Any]) -> DropMissingRowsArtifact:
        """Carry the dropna policy into the artifact.

        Reads only ``config``, so it takes ``df`` directly and needs no
        ``@fit_method`` unpacking of ``(X, y)``. Both engines apply the policy
        in the same precedence: an absolute ``threshold`` outranks the
        percentage ``missing_threshold``, which outranks ``how``.
        """
        return {
            "type": "drop_missing_rows",
            "subset": config.get("subset"),
            "how": _validate_how(config),
            "threshold": config.get("threshold"),
            "missing_threshold": config.get("missing_threshold"),
        }

fit(df, config)

Carry the dropna policy into the artifact.

Reads only config, so it takes df directly and needs no @fit_method unpacking of (X, y). Both engines apply the policy in the same precedence: an absolute threshold outranks the percentage missing_threshold, which outranks how.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

def fit(self, df: Any, config: dict[str, Any]) -> DropMissingRowsArtifact:
    """Carry the dropna policy into the artifact.

    Reads only ``config``, so it takes ``df`` directly and needs no
    ``@fit_method`` unpacking of ``(X, y)``. Both engines apply the policy
    in the same precedence: an absolute ``threshold`` outranks the
    percentage ``missing_threshold``, which outranks ``how``.
    """
    return {
        "type": "drop_missing_rows",
        "subset": config.get("subset"),
        "how": _validate_how(config),
        "threshold": config.get("threshold"),
        "missing_threshold": config.get("missing_threshold"),
    }

infer_output_schema(input_schema, config)

Pass the input schema through: rows are dropped, columns are preserved.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/drop_rows.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Pass the input schema through: rows are dropped, columns are preserved."""
    # Drops rows; column set is preserved.
    return input_schema

EllipticEnvelopeApplier

Bases: BaseApplier

Drop rows that any fitted per-column EllipticEnvelope model flags as an outlier.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

class EllipticEnvelopeApplier(BaseApplier):
    """Drop rows that any fitted per-column EllipticEnvelope model flags as an outlier."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved envelopes and skipped-column metadata without estimating covariance."""
        if not local_state_fields(
            raw,
            "elliptic_envelope",
            {"type", "models", "contamination", "warnings"},
            allow_empty=True,
        ):
            return raw
        models = raw["models"]
        if type(models) is not dict:
            raise ValueError("Fitted EllipticEnvelope models must be a dictionary.")
        for model in models.values():
            _validate_elliptic_model(model)
        validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe fitted row filtering with nonfinite rows preserved by the native applier."""
        if engine not in ("pandas", "polars"):
            return None
        EllipticEnvelopeApplier.validate_inference_state(state)
        effect = "filter" if state.get("models") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted per-column models on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        models = params.get("models", {})
        if not models:
            return X, y

        mask = pl.Series(_elliptic_mask_numpy(X, models))
        return X.filter(mask), _filter_y_polars(y, mask)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        models = params.get("models", {})
        if not models:
            return X, y
        mask = _elliptic_filter_pandas(X, models)
        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted per-column models on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted per-column models on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe fitted row filtering with nonfinite rows preserved by the native applier.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe fitted row filtering with nonfinite rows preserved by the native applier."""
    if engine not in ("pandas", "polars"):
        return None
    EllipticEnvelopeApplier.validate_inference_state(state)
    effect = "filter" if state.get("models") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect saved envelopes and skipped-column metadata without estimating covariance.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved envelopes and skipped-column metadata without estimating covariance."""
    if not local_state_fields(
        raw,
        "elliptic_envelope",
        {"type", "models", "contamination", "warnings"},
        allow_empty=True,
    ):
        return raw
    models = raw["models"]
    if type(models) is not dict:
        raise ValueError("Fitted EllipticEnvelope models must be a dictionary.")
    for model in models.values():
        _validate_elliptic_model(model)
    validate_detector_warnings(raw["warnings"])
    return raw

EllipticEnvelopeCalculator

Bases: BaseCalculator

Fit one univariate EllipticEnvelope per numeric column at the configured contamination.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

@NodeRegistry.register("EllipticEnvelope", EllipticEnvelopeApplier)
@node_meta(
    id="EllipticEnvelope",
    name="Elliptic Envelope",
    category="Preprocessing",
    description="Detect outliers in a Gaussian distributed dataset.",
    params={"contamination": 0.01, "columns": [], "random_state": DEFAULT_RANDOM_STATE},
    learns_from_data=True,
)
class EllipticEnvelopeCalculator(BaseCalculator):
    """Fit one univariate EllipticEnvelope per numeric column at the configured contamination."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # Elliptic envelope filters outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> EllipticEnvelopeArtifact:  # pylint: disable=arguments-differ
        """Fit the per-column envelopes, skipping columns with fewer than five usable samples."""
        if user_picked_no_columns(config):
            return {}

        contamination = config.get("contamination", 0.01)
        random_state = config.get("random_state", DEFAULT_RANDOM_STATE)
        # TODO(pandas-removal): same per-column coercion/dropna caveat as
        # zscore.py — EllipticEnvelope.fit itself takes numpy fine, but the
        # per-column pd.to_numeric(errors="coerce").dropna() skip-logic for
        # non-numeric/short columns needs a native-Polars equivalent first.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        models: dict[str, Any] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            if series.shape[0] < 5:
                warnings.append(f"Column '{col}': Too few samples ({series.shape[0]})")
                continue
            try:
                model = EllipticEnvelope(contamination=contamination, random_state=random_state)
                model.fit(series.to_numpy().reshape(-1, 1))
                models[col] = model
            except Exception as e:  # noqa: BLE001 - per-column fit failure is logged and recorded in warnings
                logger.warning(f"EllipticEnvelope fit failed for column {col}: {e}")
                warnings.append(f"Column '{col}': {str(e)}")

        return {
            "type": "elliptic_envelope",
            "models": models,
            "contamination": contamination,
            "warnings": warnings,
        }

fit(X, _y, config)

Fit the per-column envelopes, skipping columns with fewer than five usable samples.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> EllipticEnvelopeArtifact:  # pylint: disable=arguments-differ
    """Fit the per-column envelopes, skipping columns with fewer than five usable samples."""
    if user_picked_no_columns(config):
        return {}

    contamination = config.get("contamination", 0.01)
    random_state = config.get("random_state", DEFAULT_RANDOM_STATE)
    # TODO(pandas-removal): same per-column coercion/dropna caveat as
    # zscore.py — EllipticEnvelope.fit itself takes numpy fine, but the
    # per-column pd.to_numeric(errors="coerce").dropna() skip-logic for
    # non-numeric/short columns needs a native-Polars equivalent first.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    models: dict[str, Any] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        if series.shape[0] < 5:
            warnings.append(f"Column '{col}': Too few samples ({series.shape[0]})")
            continue
        try:
            model = EllipticEnvelope(contamination=contamination, random_state=random_state)
            model.fit(series.to_numpy().reshape(-1, 1))
            models[col] = model
        except Exception as e:  # noqa: BLE001 - per-column fit failure is logged and recorded in warnings
            logger.warning(f"EllipticEnvelope fit failed for column {col}: {e}")
            warnings.append(f"Column '{col}': {str(e)}")

    return {
        "type": "elliptic_envelope",
        "models": models,
        "contamination": contamination,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/elliptic.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # Elliptic envelope filters outlier *rows*; column set is preserved.
    return input_schema

FeatureEngineer

Orchestrates a sequence of feature engineering steps.

Examples:

>>> engineer = FeatureEngineer([])
>>> transformed, metrics = engineer.fit_transform(data)

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

class FeatureEngineer:
    """Orchestrates a sequence of feature engineering steps.

    Examples:
        >>> engineer = FeatureEngineer([])
        >>> transformed, metrics = engineer.fit_transform(data)
    """

    # Resampling steps (SMOTE/undersampling) must only ever run on the train
    # split -- applying them to test/validation would fabricate synthetic rows
    # or delete real held-out rows purely to balance classes, corrupting any
    # metrics later computed on that "held-out" data. Kept as a single shared
    # constant (rather than duplicated literals) so `transform()`, `_run_step`,
    # and `_collect_step_metrics` can't drift out of sync with each other.
    _RESAMPLING_TYPES: ClassVar[set[str]] = {"Oversampling", "Undersampling"}

    # Explicit filters apply to training and evaluation. Running them at
    # unkeyed inference would silently vanish requested input rows -- prediction
    # responses carry no row keys, so callers could never tell which inputs
    # lost their prediction. Skipping them means a null row surfaces as a
    # visible model error instead of a silent misalignment.
    _ROW_DROPPING_TYPES: ClassVar[set[str]] = {
        "Deduplicate",
        "DropMissingRows",
        "RowFilterFunction",
    }

    def __init__(
        self,
        steps_config: Sequence[PreprocessingStepConfig | dict[str, Any]],
        *,
        _validated: bool = False,
        frame_spec: FrameSpec | None = None,
        execution_options: ExecutionOptions | None = None,
    ):
        """Store the ordered step configuration, validating it unless the caller already did.

        Args:
            steps_config: Pipeline steps in execution order, as dicts or
                ``PreprocessingStepConfig`` objects.
            _validated: Skip ``validate_preprocessing_steps`` when ``True``; set by
                callers such as ``SkyulfPipeline`` that already validated the same
                structural rules via ``validate_pipeline_config``.
            frame_spec: Explicit keys and optional target for a single Spark frame.
            execution_options: Opt-in engine selection; must match the input engine.
        """
        # `Sequence` (covariant) accepts list[dict] or list[PreprocessingStepConfig].
        if not _validated:
            validate_preprocessing_steps(steps_config)
        self.steps_config = steps_config
        self.fitted_steps: list[dict[str, Any]] = []
        if frame_spec is not None and not isinstance(frame_spec, FrameSpec):
            raise TypeError("frame_spec must be FrameSpec.")
        if execution_options is not None and not isinstance(execution_options, ExecutionOptions):
            raise TypeError("execution_options must be ExecutionOptions.")
        self.frame_spec = frame_spec
        self.execution_options = execution_options
        self._spark_fitted = False
        self._portable_fitted = False

    def export_state(self) -> bytes:
        """Export supported fitted steps as bounded, versioned UTF-8 JSON bytes.

        SimpleImputer mean/constant and StandardScaler are supported. Learned
        column selection is frozen; row keys, target and runtime objects are
        supplied again when loading. The complete payload must fit the execution
        state byte budget (8 MiB by default). Callers own storage and transport.
        """
        return export_feature_state(self)

    @classmethod
    def from_state(
        cls,
        payload: bytes,
        *,
        frame_spec: FrameSpec | None = None,
        execution_options: ExecutionOptions | None = None,
    ) -> FeatureEngineer:
        """Restore supported fitted steps without fitting, I/O or creating a Spark session.

        Spark requires explicit frame_spec and execution_options. Local callers
        may omit both and let input frames select pandas or Polars. Malformed,
        oversized, unsupported or corrupt envelopes are rejected before execution.
        """
        configs, records = restore_feature_state(payload, execution_options, frame_spec)
        engineer = cls(configs, frame_spec=frame_spec, execution_options=execution_options)
        engineer.fitted_steps = records
        engineer._portable_fitted = True
        engineer._spark_fitted = (
            execution_options is not None and execution_options.engine == "spark"
        )
        return engineer

    def transform(
        self, data: pd.DataFrame | SkyulfDataFrame | Any, *, preserve_rows: bool = False
    ) -> Any:
        """Apply transformations, optionally rejecting row-count changes and temporal sorting.

        Local prediction callers set ``preserve_rows=True`` because their responses
        have no input-row provenance. Each applied step is checked immediately;
        a later step cannot conceal filtering, expansion, or a built-in temporal
        permutation. Ordinary transform and fold scoring retain their configured
        filtering and sorting behavior. Custom appliers receive count checks only.
        Spark execution always preserves key identity and optional target values;
        physical row order is not guaranteed. Consumers must align by row keys.
        """
        spec = getattr(self, "frame_spec", None)
        if use_spark(data, getattr(self, "execution_options", None), spec):
            if not getattr(self, "_spark_fitted", False):
                raise ValueError("Spark FeatureEngineer must be fitted before transform.")
            assert spec is not None
            assert self.execution_options is not None
            return transform_spark(
                data,
                self.fitted_steps,
                spec,
                state_max_bytes=self.execution_options.state_max_bytes,
            )
        current_data = data
        input_rows = prediction_row_count(unpack_pipeline_input(data)[0])
        coverage_steps: list[dict[str, Any]] = []

        for step in self._transform_steps(preserve_rows=preserve_rows):
            name = step["name"]
            transformer_type = step["type"]
            applier = step["applier"]
            artifact = step["artifact"]

            logger.debug(f"Applying step: {name} ({transformer_type})")
            rows_before = prediction_row_count(unpack_pipeline_input(current_data)[0])
            current_data = (
                _apply_prediction_step(current_data, step)
                if preserve_rows
                else applier.apply(current_data, artifact)
            )
            if not preserve_rows:
                coverage_steps.append(
                    {
                        "name": name,
                        "transformer": transformer_type,
                        **record_coverage(
                            rows_before,
                            prediction_row_count(unpack_pipeline_input(current_data)[0]),
                            step_name=f"{name} ({transformer_type})",
                        ),
                    }
                )

        if not preserve_rows:
            self.last_transform_coverage_ = record_coverage(
                input_rows,
                prediction_row_count(unpack_pipeline_input(current_data)[0]),
                coverage_steps,
            )
        return current_data

    def _transform_steps(self, *, preserve_rows: bool = True) -> list[dict[str, Any]]:
        """Apply explicit evaluation filters while retaining unkeyed prediction skips."""
        skipped = {
            "TrainTestSplitter",
            "Split",
            "feature_target_split",
            *self._RESAMPLING_TYPES,
        }
        if preserve_rows:
            skipped.update(self._ROW_DROPPING_TYPES)
        return [step for step in self.fitted_steps if step["type"] not in skipped]

    def transform_tracking_order(self, data: Any) -> tuple[Any, np.ndarray]:
        """Transform local features and report each output row's original input position.

        Only built-in temporal permutations are tracked. Row-count changes fail;
        ordinary serving continues to reject permutations via preserve_rows=True.
        """
        if use_spark(
            data, getattr(self, "execution_options", None), getattr(self, "frame_spec", None)
        ):
            raise ValueError("Positional prediction tracking requires a local frame.")
        positions = np.arange(prediction_row_count(data))
        for step in self._transform_steps():
            data, local_positions = _apply_tracked_step(data, step)
            if local_positions is not None:
                positions = positions[local_positions]
        return data, positions

    def fit_transform(
        self,
        data: pd.DataFrame | SkyulfDataFrame | Any,
        node_id_prefix="",
        *,
        target_column: str | None = None,
        on_split: Callable[[SplitDataset], None] | None = None,
        sample_weight: Any = None,
    ) -> Any:
        """Runs the pipeline on data.

        Args:
            data: Input frame, feature-target pair, or existing split dataset.
            node_id_prefix (str): Prefix for the per-step transformer identifiers.
            target_column: Execution target excluded from automatic feature selection.
            sample_weight: Optional raw positional weights, sliced by the actual
                splitter and stored in the resulting training split.
            on_split: Optional synchronous callback receiving the first actual row
                split before later transformations run. Skipped splitters and
                feature-target separation do not invoke it. The callback is not
                retained; consumers must copy or persist the payload immediately
                if they need a snapshot. Callback errors propagate to the caller.

        Returns:
            A pair containing the transformed data and per-step metrics.

        Spark execution validates all capabilities before any data action. Key
        uniqueness and per-step identity checks run distributed queries with
        bounded results; local row counts and memory metrics remain unknown.
        """
        raw_weights = prepare_pipeline_weights(data, sample_weight, self.steps_config)
        spec = getattr(self, "frame_spec", None)
        if use_spark(data, getattr(self, "execution_options", None), spec):
            return self._fit_transform_spark(data, spec, target_column, on_split)
        self._portable_fitted = False
        self.fitted_steps = []  # Reset fitted steps
        current_data = data
        split_captured = False
        metrics: dict[str, Any] = {
            "summary": {
                "fit_time": 0.0,
                "peak_memory_bytes": 0,
                "rows_in": 0,
                "rows_out": 0,
            },
            "steps": {},
        }

        for i, step in enumerate(self.steps_config):
            name = step["name"]
            transformer_type = step["transformer"]
            params = dict(step.get("params", {}))
            if target_column is not None:
                # A raw-frame target must not become an auto-selected feature.
                # Keep the caller's config untouched and use pipeline context.
                params["target_column"] = target_column
            step_metrics: dict[str, Any] = {}
            step_key = f"{i}:{name}"

            logger.info(f"Running step {i}: {name} ({transformer_type})")
            logger.debug(f"FeatureEngineer running step {i}: {name} ({transformer_type})")
            logger.debug(f"current_data type: {type(current_data)}")

            # Snapshot before for shape-delta + Winsorize value-clipping metrics
            rows_before, cols_before = get_data_stats(current_data)
            data_before = current_data

            calculator, applier = self._get_transformer_components(transformer_type)
            step_node_id = f"{node_id_prefix}_{name}"

            current_data, fitted_params, transformer_inst = self._run_step(
                transformer_type=transformer_type,
                name=name,
                calculator=calculator,
                applier=applier,
                step_node_id=step_node_id,
                current_data=current_data,
                params=params,
                sample_weight=raw_weights,
            )

            raw_weights = _step_weights(current_data, transformer_inst, raw_weights)
            record_target_labels(
                data_before, current_data, target_column, transformer_type, fitted_params
            )

            if (
                on_split is not None
                and not split_captured
                and transformer_type in {"TrainTestSplitter", "Split"}
                and not isinstance(data_before, SplitDataset)
                and isinstance(current_data, SplitDataset)
            ):
                split_captured = True
                on_split(current_data)

            logger.debug(f"Step {i} complete. New data type: {type(current_data)}")

            rows_after, cols_after = get_data_stats(current_data)
            self._collect_step_metrics(
                transformer_type=transformer_type,
                fitted_params=fitted_params,
                data_before=data_before,
                current_data=current_data,
                params=params,
                rows_before=rows_before,
                cols_before=cols_before,
                rows_after=rows_after,
                cols_after=cols_after,
                name=name,
                metrics=step_metrics,
            )

            step_record = self._step_metric_record(
                name, transformer_type, transformer_inst, rows_before, rows_after, step_metrics
            )
            metrics["steps"][step_key] = step_record

            summary = metrics["summary"]
            summary["fit_time"] += step_record["fit_time"]
            summary["peak_memory_bytes"] = max(
                summary["peak_memory_bytes"], step_record["peak_memory_bytes"]
            )
            if i == 0:
                summary["rows_in"] = step_record["rows_in"]
            summary["rows_out"] = step_record["rows_out"]

        metrics["fit_time"] = metrics["summary"]["fit_time"]
        metrics["peak_memory_bytes"] = metrics["summary"]["peak_memory_bytes"]
        metrics["rows_in"] = metrics["summary"]["rows_in"]
        metrics["rows_out"] = metrics["summary"]["rows_out"]

        self._portable_fitted = True
        self.train_sample_weight_ = raw_weights
        self.evaluation_coverage_ = deepcopy(getattr(current_data, "evaluation_coverage", {}))
        return current_data, metrics

    def _fit_transform_spark(
        self,
        data: Any,
        spec: FrameSpec | None,
        target_column: str | None,
        on_split: Callable[[SplitDataset], None] | None,
    ) -> tuple[Any, dict[str, Any]]:
        """Fit one Spark frame and publish learned records after native execution succeeds."""
        assert spec is not None
        if target_column is not None and target_column != spec.target:
            raise ValueError("target_column conflicts with frame_spec.target.")
        if on_split is not None:
            raise ValueError("Spark FE uses one frame; on_split is unsupported.")
        assert self.execution_options is not None
        result, metrics, records = fit_spark(
            data,
            self.steps_config,
            spec,
            state_max_bytes=self.execution_options.state_max_bytes,
        )
        self.fitted_steps = records
        self._spark_fitted = True
        self._portable_fitted = True
        return result, metrics

    @staticmethod
    def _step_metric_record(
        name: str,
        transformer_type: str,
        transformer_inst: Any,
        rows_before: int,
        rows_after: int,
        details: dict[str, Any],
    ) -> dict[str, Any]:
        """Capture a step's measurements, falling back to observed row counts."""
        return {
            "name": name,
            "transformer": transformer_type,
            "fit_time": (
                getattr(transformer_inst, "fit_time", 0.0) if transformer_inst is not None else 0.0
            ),
            "peak_memory_bytes": (
                getattr(transformer_inst, "peak_memory_bytes", 0)
                if transformer_inst is not None
                else 0
            ),
            "rows_in": (
                getattr(transformer_inst, "rows_in", rows_before)
                if transformer_inst is not None
                else rows_before
            ),
            "rows_out": (
                getattr(transformer_inst, "rows_out", rows_after)
                if transformer_inst is not None
                else rows_after
            ),
            "details": details,
        }

    # ------------------------------------------------------------------
    # Step execution
    # ------------------------------------------------------------------

    def _run_step(
        self,
        *,
        transformer_type: str,
        name: str,
        calculator: Any,
        applier: Any,
        step_node_id: str,
        current_data: Any,
        params: dict[str, Any],
        sample_weight: Any = None,
    ) -> tuple:  # Returns (data, params, transformer)
        """Execute one pipeline step. Returns (new_data, fitted_params).

        Splitters change the data structure (DataFrame -> SplitDataset / (X, y)),
        so they bypass StatefulTransformer; everything else goes through the
        standard fit_transform wrapper and is appended to fitted_steps.
        """
        transformer = StatefulTransformer(
            calculator,
            applier,
            step_node_id,
            apply_on_test=transformer_type not in self._RESAMPLING_TYPES,
            apply_on_validation=transformer_type not in self._RESAMPLING_TYPES,
        )
        fitted_params: dict[str, Any] = {}

        if transformer_type in {"TrainTestSplitter", "Split"}:
            logger.debug("Handling TrainTestSplitter")
            # A raw (unwrapped) polars DataFrame satisfies neither `pd.DataFrame`
            # nor the `SkyulfDataFrame` protocol (it has no `.copy()`, uses
            # `.clone()` instead) even though it's a fully-supported "not yet
            # split" input -- `SplitCalculator`/`SplitApplier` already handle it
            # correctly via their own polars round-trip. Without this extra
            # check, a raw polars DataFrame passed straight to
            # `SkyulfPipeline.fit()` (the advertised polars-native usage) would
            # silently skip the split entirely and fit/evaluate on the whole
            # dataset with no held-out test set.
            if isinstance(current_data, pd.DataFrame | SkyulfDataFrame | tuple | pl.DataFrame):
                params = calculator.fit(current_data, params)
                weight_kwargs = {} if sample_weight is None else {"sample_weight": sample_weight}
                current_data = applier.apply(current_data, params, **weight_kwargs)
            else:
                logger.debug(f"Skipping TrainTestSplitter. current_data is {type(current_data)}")
                logger.warning(
                    "Attempting to split an already split dataset. Skipping TrainTestSplitter."
                )
            return current_data, fitted_params, None

        if transformer_type == "feature_target_split":
            logger.debug("Handling feature_target_split")
            params = calculator.fit(current_data, params)
            current_data = applier.apply(current_data, params)
            return current_data, fitted_params, None

        logger.debug("Handling standard transformer via StatefulTransformer")
        if sample_weight is not None and not isinstance(current_data, SplitDataset):
            current_data, sample_weight = transformer.fit_transform_weighted(
                current_data, params, sample_weight
            )
        else:
            current_data = transformer.fit_transform(current_data, params)
        transformer.train_sample_weight_ = sample_weight
        fitted_params = transformer.params
        self.fitted_steps.append(
            {
                "name": name,
                "type": transformer_type,
                "applier": applier,
                "artifact": fitted_params,
                "params": dict(params),
            }
        )
        return current_data, fitted_params, transformer

    # ------------------------------------------------------------------
    # Metrics collection
    # ------------------------------------------------------------------

    # Transformer-type groups, kept as class constants so dispatch is data-driven.
    _IMPUTATION_TYPES: ClassVar[set[str]] = {"SimpleImputer", "KNNImputer", "IterativeImputer"}
    _FEATURE_SELECTION_TYPES: ClassVar[set[str]] = {
        "feature_selection",
        "UnivariateSelection",
        "ModelBasedSelection",
        "VarianceThreshold",
    }
    _SCALING_TYPES: ClassVar[set[str]] = {
        "StandardScaler",
        "MinMaxScaler",
        "RobustScaler",
        "MaxAbsScaler",
    }
    _OUTLIER_TYPES: ClassVar[set[str]] = {"IQR", "Winsorize", "ZScore", "EllipticEnvelope"}
    _BUCKETING_TYPES: ClassVar[set[str]] = {
        "GeneralBinning",
        "EqualWidthBinning",
        "EqualFrequencyBinning",
        "CustomBinning",
        "KBinsDiscretizer",
    }
    _FEATURE_GEN_TYPES: ClassVar[set[str]] = {"FeatureMath", "FeatureGenerationNode"}
    _ROW_DROP_TYPES: ClassVar[set[str]] = {
        "DropMissingRows",
        "Deduplicate",
        "IQR",
        "ZScore",
        "EllipticEnvelope",
        "Winsorize",
    }
    _ENCODER_TYPES: ClassVar[set[str]] = {
        "OneHotEncoder",
        "LabelEncoder",
        "OrdinalEncoder",
        "TargetEncoder",
        "HashEncoder",
        "DummyEncoder",
    }

    def _collect_step_metrics(
        self,
        *,
        transformer_type: str,
        fitted_params: dict[str, Any],
        data_before: Any,
        current_data: Any,
        params: dict[str, Any],
        rows_before: int,
        cols_before: Any,
        rows_after: int,
        cols_after: Any,
        name: str,
        metrics: dict[str, Any],
    ) -> None:
        """Populate one step record's details dict with node-specific metrics."""
        try:
            if fitted_params:
                self._metrics_from_fitted_params(
                    transformer_type, fitted_params, data_before, current_data, metrics
                )
        except Exception as e:  # noqa: BLE001 - step metrics are best-effort; logged
            logger.warning(f"Failed to retrieve metrics for step {name}: {e}")

        if transformer_type in self._RESAMPLING_TYPES:
            self._metrics_resampling(current_data, params, metrics)

        if rows_after > 0 or cols_after:
            self._metrics_shape_change(
                transformer_type,
                data_before,
                current_data,
                params,
                rows_before,
                cols_before,
                rows_after,
                cols_after,
                metrics,
            )

    @staticmethod
    def _copy_present_keys(
        fitted_params: dict[str, Any], metrics: dict[str, Any], keys: tuple[str, ...]
    ) -> None:
        """Copy fitted params into a single step's details dict when present."""
        for key in keys:
            if key in fitted_params:
                metrics[key] = fitted_params[key]

    # Each rule maps a set of transformer types to the fitted_params/metrics key
    # to copy over when that transformer type produced it. Keys are the same on
    # both sides for every current rule.
    _OUTLIER_METRIC_RULES: tuple[tuple[frozenset[str], str], ...] = (
        (frozenset({"IQR", "Winsorize"}), "bounds"),
        (frozenset({"ZScore"}), "stats"),
        (frozenset({"EllipticEnvelope"}), "contamination"),
    )

    def _apply_outlier_metrics(
        self, transformer_type: str, fitted_params: dict[str, Any], metrics: dict[str, Any]
    ) -> None:
        """Populate one step's outlier details (warnings/bounds/stats/contamination)."""
        if transformer_type in self._OUTLIER_TYPES and "warnings" in fitted_params:
            metrics["warnings"] = fitted_params["warnings"]
        for types, key in self._OUTLIER_METRIC_RULES:
            if transformer_type in types and key in fitted_params:
                metrics[key] = fitted_params[key]

    def _apply_feature_gen_metrics(
        self,
        fitted_params: dict[str, Any],
        data_before: Any,
        current_data: Any,
        metrics: dict[str, Any],
    ) -> None:
        """Populate one step's generated-feature details and operation metadata."""
        if "operations" in fitted_params:
            metrics["operations_count"] = len(fitted_params["operations"])
            metrics["operations"] = fitted_params["operations"]
        new_cols = self._diff_generated_columns(data_before, current_data)
        if new_cols is not None:
            metrics["generated_features"] = new_cols

    def _metrics_from_fitted_params(
        self,
        transformer_type: str,
        fitted_params: dict[str, Any],
        data_before: Any,
        current_data: Any,
        metrics: dict[str, Any],
    ) -> None:
        """Populate one step's details from fitted parameters and derived artifacts."""
        if transformer_type in self._IMPUTATION_TYPES:
            self._copy_present_keys(
                fitted_params, metrics, ("missing_counts", "total_missing", "fill_values")
            )

        if transformer_type in self._FEATURE_SELECTION_TYPES:
            self._copy_present_keys(
                fitted_params,
                metrics,
                (
                    "feature_scores",
                    "p_values",
                    "feature_importances",
                    "variances",
                    "ranking",
                    "selected_columns",
                ),
            )

        if transformer_type in self._SCALING_TYPES:
            self._copy_present_keys(
                fitted_params,
                metrics,
                (
                    "mean",
                    "scale",
                    "var",
                    "min",
                    "data_min",
                    "data_max",
                    "center",
                    "max_abs",
                    "columns",
                ),
            )

        self._apply_outlier_metrics(transformer_type, fitted_params, metrics)

        if transformer_type in self._BUCKETING_TYPES:
            self._copy_present_keys(fitted_params, metrics, ("bin_edges", "n_bins"))

        if transformer_type in self._FEATURE_GEN_TYPES:
            self._apply_feature_gen_metrics(fitted_params, data_before, current_data, metrics)

    @staticmethod
    def _columns_diff_if_dataframes(before: Any, after: Any):
        """Return the column-set difference if both objects are DataFrames, else None."""
        if isinstance(before, pd.DataFrame | SkyulfDataFrame) and isinstance(
            after, pd.DataFrame | SkyulfDataFrame
        ):
            return list(set(after.columns) - set(before.columns))
        return None

    @classmethod
    def _diff_generated_columns_split_dataset(
        cls, data_before: SplitDataset, current_data: SplitDataset
    ):
        """Diff columns for the SplitDataset case, handling both DataFrame and (X, y) train shapes."""
        before_train, after_train = data_before.train, current_data.train
        diff = cls._columns_diff_if_dataframes(before_train, after_train)
        if diff is not None:
            return diff
        if isinstance(before_train, tuple) and isinstance(after_train, tuple):
            x_before, _ = before_train
            x_after, _ = after_train
            return cls._columns_diff_if_dataframes(x_before, x_after)
        return None

    @classmethod
    def _diff_generated_columns(cls, data_before: Any, current_data: Any):
        """Return the set of newly added columns between two pipeline data objects.

        Handles plain DataFrames, SplitDatasets of DataFrames, and (X, y) tuple variants.
        Returns None if the structures don't allow a meaningful diff.
        """
        diff = cls._columns_diff_if_dataframes(data_before, current_data)
        if diff is not None:
            return diff

        if isinstance(data_before, SplitDataset) and isinstance(current_data, SplitDataset):
            return cls._diff_generated_columns_split_dataset(data_before, current_data)
        return None

    @staticmethod
    def _extract_y_from_split_dataset(current_data: SplitDataset, params: dict[str, Any]):
        """Pull the target Series out of a SplitDataset's train split (tuple or DataFrame form)."""
        if isinstance(current_data.train, tuple):
            _, y_res = current_data.train
            return y_res
        if isinstance(current_data.train, (pd.DataFrame, SkyulfDataFrame)):
            target_col = params.get("target_column")
            if target_col and target_col in current_data.train.columns:
                return current_data.train[target_col]
        return None

    @classmethod
    def _extract_y_for_resampling(cls, current_data: Any, params: dict[str, Any]):
        """Pull the target Series out of whatever shape the resampler produced."""
        if isinstance(current_data, SplitDataset):
            return cls._extract_y_from_split_dataset(current_data, params)
        if isinstance(current_data, tuple):
            _, y_res = current_data
            return y_res
        if isinstance(current_data, pd.DataFrame | SkyulfDataFrame):
            target_col = params.get("target_column")
            if target_col and target_col in current_data.columns:
                return current_data[target_col]
        return None

    def _metrics_resampling(
        self, current_data: Any, params: dict[str, Any], metrics: dict[str, Any]
    ) -> None:
        """Populate one resampling step's class-balance details."""
        try:
            y_res: Any = self._extract_y_for_resampling(current_data, params)
            if y_res is None:
                return
            if hasattr(y_res, "to_pandas"):
                y_res = y_res.to_pandas()
            counts = y_res.value_counts().to_dict()
            metrics["class_counts"] = {str(k): int(v) for k, v in counts.items()}
            metrics["total_samples"] = int(len(y_res))
        except Exception as e:  # noqa: BLE001 - resampling metrics are best-effort; logged
            logger.warning(f"Failed to calculate resampling metrics: {e}")

    @staticmethod
    def _to_pandas_if_needed(obj: Any) -> Any:
        """Convert obj to pandas via to_pandas() if it supports that, otherwise return as-is."""
        return obj.to_pandas() if hasattr(obj, "to_pandas") else obj

    @classmethod
    def _count_diff_cells(cls, a: Any, b: Any, types: tuple[type, ...]) -> int:
        """Count differing cells between two objects of the given types with matching shape."""
        a = cls._to_pandas_if_needed(a)
        b = cls._to_pandas_if_needed(b)
        if isinstance(a, types) and isinstance(b, types) and a.shape == b.shape:
            return int(a.ne(b).sum().sum())
        return 0

    @classmethod
    def _count_winsorize_tuple_diffs(cls, d1: tuple, d2: tuple) -> int:
        """Count differing cells across the X/y halves of two (X, y) tuple pairs."""
        diffs = cls._count_diff_cells(d1[0], d2[0], (pd.DataFrame,))
        diffs += cls._count_diff_cells(d1[1], d2[1], (pd.DataFrame, pd.Series))
        return diffs

    @classmethod
    def _count_winsorize_diffs(cls, d1: Any, d2: Any) -> int:
        """Count cells that differ between two data objects, for Winsorize clipping metric."""
        d1 = cls._to_pandas_if_needed(d1)
        d2 = cls._to_pandas_if_needed(d2)

        if isinstance(d1, pd.DataFrame) and isinstance(d2, pd.DataFrame):
            return cls._count_diff_cells(d1, d2, (pd.DataFrame,))

        if isinstance(d1, tuple) and isinstance(d2, tuple) and len(d1) == 2 and len(d2) == 2:
            return cls._count_winsorize_tuple_diffs(d1, d2)
        return 0

    def _metrics_winsorize_clipped(
        self, data_before: Any, current_data: Any, metrics: dict[str, Any]
    ) -> None:
        """Populate one Winsorize step's clipped-value count detail."""
        try:
            clipped_count = 0
            if isinstance(data_before, (pd.DataFrame, SkyulfDataFrame)) and isinstance(
                current_data, (pd.DataFrame, SkyulfDataFrame)
            ):
                clipped_count = self._count_winsorize_diffs(data_before, current_data)
            elif isinstance(data_before, SplitDataset) and isinstance(current_data, SplitDataset):
                clipped_count += self._count_winsorize_diffs(data_before.train, current_data.train)
                clipped_count += self._count_winsorize_diffs(data_before.test, current_data.test)
                clipped_count += self._count_winsorize_diffs(
                    data_before.validation, current_data.validation
                )
            metrics["values_clipped"] = clipped_count
        except Exception as e:  # noqa: BLE001 - winsorize metrics are best-effort; logged
            logger.warning(f"Failed to calculate values_clipped for Winsorize: {e}")

    def _metrics_shape_change(
        self,
        transformer_type: str,
        data_before: Any,
        current_data: Any,
        params: dict[str, Any],
        rows_before: int,
        cols_before: Any,
        rows_after: int,
        cols_after: Any,
        metrics: dict[str, Any],
    ) -> None:
        """Populate one step's shape-change details such as dropped rows/columns."""
        if transformer_type in self._ROW_DROP_TYPES:
            dropped = rows_before - rows_after
            metrics[f"{transformer_type}_rows_removed"] = dropped
            metrics[f"{transformer_type}_rows_remaining"] = rows_after
            metrics[f"{transformer_type}_rows_total"] = rows_before
            metrics["rows_removed"] = dropped
            metrics["rows_total"] = rows_before
            if transformer_type == "Winsorize":
                self._metrics_winsorize_clipped(data_before, current_data, metrics)

        if transformer_type == "MissingIndicator":
            new_cols_set = cols_after - cols_before
            metrics["missing_indicators_created"] = len(new_cols_set)
            metrics["missing_indicators_columns"] = list(new_cols_set)

        if transformer_type in {"DropMissingColumns", "feature_selection"}:
            dropped_cols_set = cols_before - cols_after
            metrics["dropped_columns"] = list(dropped_cols_set)
            metrics["dropped_columns_count"] = len(dropped_cols_set)

        if transformer_type in self._ENCODER_TYPES:
            new_cols_set = cols_after - cols_before
            metrics["new_features_count"] = len(new_cols_set)
            metrics["encoded_columns_count"] = len(params.get("columns", []))
            if "categories_count" in params:
                metrics["categories_count"] = params["categories_count"]
            if "classes_count" in params:
                metrics["classes_count"] = params["classes_count"]

    def _get_transformer_components(self, type_name: str):
        try:
            return (
                NodeRegistry.get_calculator(type_name)(),
                NodeRegistry.get_applier(type_name)(),
            )
        except ValueError as exc:
            raise ValueError(f"Unknown transformer type: {type_name}. {exc}") from exc

__init__(steps_config, *, _validated=False, frame_spec=None, execution_options=None)

Store the ordered step configuration, validating it unless the caller already did.

Parameters:

Name	Type	Description	Default
steps_config	Sequence[PreprocessingStepConfig | dict[str, Any]]	Pipeline steps in execution order, as dicts or PreprocessingStepConfig objects.	required
_validated	bool	Skip validate_preprocessing_steps when True; set by callers such as SkyulfPipeline that already validated the same structural rules via validate_pipeline_config.	False
frame_spec	FrameSpec | None	Explicit keys and optional target for a single Spark frame.	None
execution_options	ExecutionOptions | None	Opt-in engine selection; must match the input engine.	None

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

def __init__(
    self,
    steps_config: Sequence[PreprocessingStepConfig | dict[str, Any]],
    *,
    _validated: bool = False,
    frame_spec: FrameSpec | None = None,
    execution_options: ExecutionOptions | None = None,
):
    """Store the ordered step configuration, validating it unless the caller already did.

    Args:
        steps_config: Pipeline steps in execution order, as dicts or
            ``PreprocessingStepConfig`` objects.
        _validated: Skip ``validate_preprocessing_steps`` when ``True``; set by
            callers such as ``SkyulfPipeline`` that already validated the same
            structural rules via ``validate_pipeline_config``.
        frame_spec: Explicit keys and optional target for a single Spark frame.
        execution_options: Opt-in engine selection; must match the input engine.
    """
    # `Sequence` (covariant) accepts list[dict] or list[PreprocessingStepConfig].
    if not _validated:
        validate_preprocessing_steps(steps_config)
    self.steps_config = steps_config
    self.fitted_steps: list[dict[str, Any]] = []
    if frame_spec is not None and not isinstance(frame_spec, FrameSpec):
        raise TypeError("frame_spec must be FrameSpec.")
    if execution_options is not None and not isinstance(execution_options, ExecutionOptions):
        raise TypeError("execution_options must be ExecutionOptions.")
    self.frame_spec = frame_spec
    self.execution_options = execution_options
    self._spark_fitted = False
    self._portable_fitted = False

export_state()

Export supported fitted steps as bounded, versioned UTF-8 JSON bytes.

SimpleImputer mean/constant and StandardScaler are supported. Learned column selection is frozen; row keys, target and runtime objects are supplied again when loading. The complete payload must fit the execution state byte budget (8 MiB by default). Callers own storage and transport.

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

def export_state(self) -> bytes:
    """Export supported fitted steps as bounded, versioned UTF-8 JSON bytes.

    SimpleImputer mean/constant and StandardScaler are supported. Learned
    column selection is frozen; row keys, target and runtime objects are
    supplied again when loading. The complete payload must fit the execution
    state byte budget (8 MiB by default). Callers own storage and transport.
    """
    return export_feature_state(self)

fit_transform(data, node_id_prefix='', *, target_column=None, on_split=None, sample_weight=None)

Runs the pipeline on data.

Parameters:

Name	Type	Description	Default
data	DataFrame | SkyulfDataFrame | Any	Input frame, feature-target pair, or existing split dataset.	required
node_id_prefix	str	Prefix for the per-step transformer identifiers.	''
target_column	str | None	Execution target excluded from automatic feature selection.	None
sample_weight	Any	Optional raw positional weights, sliced by the actual splitter and stored in the resulting training split.	None
on_split	Callable[[SplitDataset], None] | None	Optional synchronous callback receiving the first actual row split before later transformations run. Skipped splitters and feature-target separation do not invoke it. The callback is not retained; consumers must copy or persist the payload immediately if they need a snapshot. Callback errors propagate to the caller.	None

Returns:

Type	Description
Any	A pair containing the transformed data and per-step metrics.

Spark execution validates all capabilities before any data action. Key uniqueness and per-step identity checks run distributed queries with bounded results; local row counts and memory metrics remain unknown.

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

def fit_transform(
    self,
    data: pd.DataFrame | SkyulfDataFrame | Any,
    node_id_prefix="",
    *,
    target_column: str | None = None,
    on_split: Callable[[SplitDataset], None] | None = None,
    sample_weight: Any = None,
) -> Any:
    """Runs the pipeline on data.

    Args:
        data: Input frame, feature-target pair, or existing split dataset.
        node_id_prefix (str): Prefix for the per-step transformer identifiers.
        target_column: Execution target excluded from automatic feature selection.
        sample_weight: Optional raw positional weights, sliced by the actual
            splitter and stored in the resulting training split.
        on_split: Optional synchronous callback receiving the first actual row
            split before later transformations run. Skipped splitters and
            feature-target separation do not invoke it. The callback is not
            retained; consumers must copy or persist the payload immediately
            if they need a snapshot. Callback errors propagate to the caller.

    Returns:
        A pair containing the transformed data and per-step metrics.

    Spark execution validates all capabilities before any data action. Key
    uniqueness and per-step identity checks run distributed queries with
    bounded results; local row counts and memory metrics remain unknown.
    """
    raw_weights = prepare_pipeline_weights(data, sample_weight, self.steps_config)
    spec = getattr(self, "frame_spec", None)
    if use_spark(data, getattr(self, "execution_options", None), spec):
        return self._fit_transform_spark(data, spec, target_column, on_split)
    self._portable_fitted = False
    self.fitted_steps = []  # Reset fitted steps
    current_data = data
    split_captured = False
    metrics: dict[str, Any] = {
        "summary": {
            "fit_time": 0.0,
            "peak_memory_bytes": 0,
            "rows_in": 0,
            "rows_out": 0,
        },
        "steps": {},
    }

    for i, step in enumerate(self.steps_config):
        name = step["name"]
        transformer_type = step["transformer"]
        params = dict(step.get("params", {}))
        if target_column is not None:
            # A raw-frame target must not become an auto-selected feature.
            # Keep the caller's config untouched and use pipeline context.
            params["target_column"] = target_column
        step_metrics: dict[str, Any] = {}
        step_key = f"{i}:{name}"

        logger.info(f"Running step {i}: {name} ({transformer_type})")
        logger.debug(f"FeatureEngineer running step {i}: {name} ({transformer_type})")
        logger.debug(f"current_data type: {type(current_data)}")

        # Snapshot before for shape-delta + Winsorize value-clipping metrics
        rows_before, cols_before = get_data_stats(current_data)
        data_before = current_data

        calculator, applier = self._get_transformer_components(transformer_type)
        step_node_id = f"{node_id_prefix}_{name}"

        current_data, fitted_params, transformer_inst = self._run_step(
            transformer_type=transformer_type,
            name=name,
            calculator=calculator,
            applier=applier,
            step_node_id=step_node_id,
            current_data=current_data,
            params=params,
            sample_weight=raw_weights,
        )

        raw_weights = _step_weights(current_data, transformer_inst, raw_weights)
        record_target_labels(
            data_before, current_data, target_column, transformer_type, fitted_params
        )

        if (
            on_split is not None
            and not split_captured
            and transformer_type in {"TrainTestSplitter", "Split"}
            and not isinstance(data_before, SplitDataset)
            and isinstance(current_data, SplitDataset)
        ):
            split_captured = True
            on_split(current_data)

        logger.debug(f"Step {i} complete. New data type: {type(current_data)}")

        rows_after, cols_after = get_data_stats(current_data)
        self._collect_step_metrics(
            transformer_type=transformer_type,
            fitted_params=fitted_params,
            data_before=data_before,
            current_data=current_data,
            params=params,
            rows_before=rows_before,
            cols_before=cols_before,
            rows_after=rows_after,
            cols_after=cols_after,
            name=name,
            metrics=step_metrics,
        )

        step_record = self._step_metric_record(
            name, transformer_type, transformer_inst, rows_before, rows_after, step_metrics
        )
        metrics["steps"][step_key] = step_record

        summary = metrics["summary"]
        summary["fit_time"] += step_record["fit_time"]
        summary["peak_memory_bytes"] = max(
            summary["peak_memory_bytes"], step_record["peak_memory_bytes"]
        )
        if i == 0:
            summary["rows_in"] = step_record["rows_in"]
        summary["rows_out"] = step_record["rows_out"]

    metrics["fit_time"] = metrics["summary"]["fit_time"]
    metrics["peak_memory_bytes"] = metrics["summary"]["peak_memory_bytes"]
    metrics["rows_in"] = metrics["summary"]["rows_in"]
    metrics["rows_out"] = metrics["summary"]["rows_out"]

    self._portable_fitted = True
    self.train_sample_weight_ = raw_weights
    self.evaluation_coverage_ = deepcopy(getattr(current_data, "evaluation_coverage", {}))
    return current_data, metrics

from_state(payload, *, frame_spec=None, execution_options=None) classmethod

Restore supported fitted steps without fitting, I/O or creating a Spark session.

Spark requires explicit frame_spec and execution_options. Local callers may omit both and let input frames select pandas or Polars. Malformed, oversized, unsupported or corrupt envelopes are rejected before execution.

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

@classmethod
def from_state(
    cls,
    payload: bytes,
    *,
    frame_spec: FrameSpec | None = None,
    execution_options: ExecutionOptions | None = None,
) -> FeatureEngineer:
    """Restore supported fitted steps without fitting, I/O or creating a Spark session.

    Spark requires explicit frame_spec and execution_options. Local callers
    may omit both and let input frames select pandas or Polars. Malformed,
    oversized, unsupported or corrupt envelopes are rejected before execution.
    """
    configs, records = restore_feature_state(payload, execution_options, frame_spec)
    engineer = cls(configs, frame_spec=frame_spec, execution_options=execution_options)
    engineer.fitted_steps = records
    engineer._portable_fitted = True
    engineer._spark_fitted = (
        execution_options is not None and execution_options.engine == "spark"
    )
    return engineer

transform(data, *, preserve_rows=False)

Apply transformations, optionally rejecting row-count changes and temporal sorting.

Local prediction callers set preserve_rows=True because their responses have no input-row provenance. Each applied step is checked immediately; a later step cannot conceal filtering, expansion, or a built-in temporal permutation. Ordinary transform and fold scoring retain their configured filtering and sorting behavior. Custom appliers receive count checks only. Spark execution always preserves key identity and optional target values; physical row order is not guaranteed. Consumers must align by row keys.

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

def transform(
    self, data: pd.DataFrame | SkyulfDataFrame | Any, *, preserve_rows: bool = False
) -> Any:
    """Apply transformations, optionally rejecting row-count changes and temporal sorting.

    Local prediction callers set ``preserve_rows=True`` because their responses
    have no input-row provenance. Each applied step is checked immediately;
    a later step cannot conceal filtering, expansion, or a built-in temporal
    permutation. Ordinary transform and fold scoring retain their configured
    filtering and sorting behavior. Custom appliers receive count checks only.
    Spark execution always preserves key identity and optional target values;
    physical row order is not guaranteed. Consumers must align by row keys.
    """
    spec = getattr(self, "frame_spec", None)
    if use_spark(data, getattr(self, "execution_options", None), spec):
        if not getattr(self, "_spark_fitted", False):
            raise ValueError("Spark FeatureEngineer must be fitted before transform.")
        assert spec is not None
        assert self.execution_options is not None
        return transform_spark(
            data,
            self.fitted_steps,
            spec,
            state_max_bytes=self.execution_options.state_max_bytes,
        )
    current_data = data
    input_rows = prediction_row_count(unpack_pipeline_input(data)[0])
    coverage_steps: list[dict[str, Any]] = []

    for step in self._transform_steps(preserve_rows=preserve_rows):
        name = step["name"]
        transformer_type = step["type"]
        applier = step["applier"]
        artifact = step["artifact"]

        logger.debug(f"Applying step: {name} ({transformer_type})")
        rows_before = prediction_row_count(unpack_pipeline_input(current_data)[0])
        current_data = (
            _apply_prediction_step(current_data, step)
            if preserve_rows
            else applier.apply(current_data, artifact)
        )
        if not preserve_rows:
            coverage_steps.append(
                {
                    "name": name,
                    "transformer": transformer_type,
                    **record_coverage(
                        rows_before,
                        prediction_row_count(unpack_pipeline_input(current_data)[0]),
                        step_name=f"{name} ({transformer_type})",
                    ),
                }
            )

    if not preserve_rows:
        self.last_transform_coverage_ = record_coverage(
            input_rows,
            prediction_row_count(unpack_pipeline_input(current_data)[0]),
            coverage_steps,
        )
    return current_data

transform_tracking_order(data)

Transform local features and report each output row's original input position.

Only built-in temporal permutations are tracked. Row-count changes fail; ordinary serving continues to reject permutations via preserve_rows=True.

Source code in skyulf-core/skyulf/preprocessing/pipeline.py

def transform_tracking_order(self, data: Any) -> tuple[Any, np.ndarray]:
    """Transform local features and report each output row's original input position.

    Only built-in temporal permutations are tracked. Row-count changes fail;
    ordinary serving continues to reject permutations via preserve_rows=True.
    """
    if use_spark(
        data, getattr(self, "execution_options", None), getattr(self, "frame_spec", None)
    ):
        raise ValueError("Positional prediction tracking requires a local frame.")
    positions = np.arange(prediction_row_count(data))
    for step in self._transform_steps():
        data, local_positions = _apply_tracked_step(data, step)
        if local_positions is not None:
            positions = positions[local_positions]
    return data, positions

FeatureEngineerFoldAdapter

Re-runs a preprocessing step chain inside each CV/tuning fold.

Parameters:

Name	Type	Description	Default
steps_config	list[dict[str, Any]]	The upstream Feature-Engineering node's step list (plain dicts, validated by validate_preprocessing_steps). Splitter steps are filtered out automatically.	required
target_column	str	Name of the target, kept to reject payloads where X still embeds it — target-aware steps (target encoders, resampling, label encoding of the target) receive y through the (X, y) payload the same way they do downstream of a real splitter.	required

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

class FeatureEngineerFoldAdapter:
    """Re-runs a preprocessing step chain inside each CV/tuning fold.

    Args:
        steps_config: The upstream Feature-Engineering node's step list
            (plain dicts, validated by ``validate_preprocessing_steps``).
            Splitter steps are filtered out automatically.
        target_column: Name of the target, kept to reject payloads where X
            still embeds it — target-aware steps (target encoders, resampling,
            label encoding of the target) receive ``y`` through the ``(X, y)``
            payload the same way they do downstream of a real splitter.
    """

    def __init__(self, steps_config: list[dict[str, Any]], target_column: str):
        """Store the chain minus any splitter steps and validate it eagerly.

        Splitter steps are filtered out because they already ran upstream of the
        fold boundary. Unlike :class:`MergedBranchFoldAdapter`, row-changing
        steps are accepted here and ``changes_row_count`` merely reports the
        chain's nature: a single branch has no column-wise merge to
        desynchronise.

        Raises:
            ValueError: If a step names a transformer absent from the registry,
                or if the trial :class:`FeatureEngineer` rejects the chain.
        """
        self._steps_config = [
            step for step in steps_config if step.get("transformer") not in SPLITTER_STEP_TYPES
        ]
        self._target_column = target_column
        # True when any step reshapes the rows/target (resampling, row
        # drops); documents the chain's nature for callers that need it.
        self.changes_row_count = any(step_changes_row_count(step) for step in self._steps_config)
        # Validate eagerly (unknown transformer names, bad params) so a
        # misconfigured chain fails at construction, not mid-fold.
        # validate_preprocessing_steps does not check registry membership,
        # so resolve each step's calculator explicitly.
        FeatureEngineer(self._steps_config)
        for step in self._steps_config:
            NodeRegistry.get_calculator(step["transformer"])
        self._engineer: FeatureEngineer | None = None

    def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
        """Fit a fresh engineer on this fold's ``(X, y)`` payload and return it.

        A new :class:`FeatureEngineer` is built on every call, so nothing fitted
        on one fold survives into the next.

        Raises:
            ValueError: If ``X`` still embeds the configured target column.
        """
        self._validate_payload(X)
        engineer = FeatureEngineer(self._steps_config)
        transformed, _metrics = engineer.fit_transform(
            (X, y), sample_weight=sample_weight, target_column=self._target_column
        )
        self._engineer = engineer
        self.train_sample_weight_ = engineer.train_sample_weight_
        return transformed

    @property
    def temporal_history_columns(self) -> set[str]:
        """Report ordering/entity columns needed until carry features are computed."""
        columns = set()
        for step in self._steps_config:
            params = step.get("params", {})
            if params.get("history_mode") == "carry":
                columns.update([params.get("sort_by"), *(params.get("group_by") or [])])
        return columns - {None}

    def retain_split_metadata(self, column: str) -> None:
        """Drop retained split metadata after preprocessing, including during serving."""
        step = {
            "name": f"exclude_split_metadata_{column}",
            "transformer": "DropMissingColumns",
            "params": {"columns": [column], "missing_threshold": None},
        }
        if step not in self._steps_config:
            self._steps_config = [*self._steps_config, step]

    def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
        """Apply the engineer fitted by the last :meth:`fit_transform`.

        Always returns an ``(X, y)`` pair, re-pairing when an applier handed
        back a bare frame, so callers can rely on the ``FoldPreprocessor``
        protocol shape.

        Raises:
            RuntimeError: If called before :meth:`fit_transform`.
            ValueError: If ``X`` still embeds the configured target column.
        """
        if self._engineer is None:
            raise RuntimeError("transform() called before fit_transform()")
        self._validate_payload(X)
        # At transform/inference time no step needs the target (target-aware
        # encoders use their fitted artifact; splitters/resampling are skipped).
        # Feed a bare frame when y is absent — a ``(X, None)`` tuple would trip
        # ``pack_pipeline_output``'s tuple-shape-lost diagnostic on every step.
        payload = (X, y) if y is not None else X
        transformed = self._engineer.transform(payload)
        self.last_transform_coverage_ = deepcopy(self._engineer.last_transform_coverage_)
        # Some appliers return a bare frame instead of the ``(X, y)`` payload;
        # re-pair so callers always get the FoldPreprocessor protocol shape.
        if not (isinstance(transformed, tuple) and len(transformed) == 2):
            transformed = (transformed, y)
        return transformed

    def _validate_payload(self, X: Any) -> None:
        if hasattr(X, "columns") and self._target_column in X.columns:
            raise ValueError(f"target column '{self._target_column}' already present in X")

    def original_target_labels(self, labels: Any) -> Any:
        """Decode the recorded fitted target chain independently of row permutations."""
        if self._engineer is None:
            raise RuntimeError("Target decoding called before fit_transform()")
        return original_target_labels(self._engineer.fitted_steps, labels)

    def transform_tracking_order(self, X: Any) -> tuple[Any, Any]:
        """Expose temporal row lineage for searcher prediction and nested thresholds."""
        if self._engineer is None:
            raise RuntimeError("transform() called before fit_transform()")
        self._validate_payload(X)
        return self._engineer.transform_tracking_order(X)

temporal_history_columns property

Report ordering/entity columns needed until carry features are computed.

__init__(steps_config, target_column)

Store the chain minus any splitter steps and validate it eagerly.

Splitter steps are filtered out because they already ran upstream of the fold boundary. Unlike :class:MergedBranchFoldAdapter, row-changing steps are accepted here and changes_row_count merely reports the chain's nature: a single branch has no column-wise merge to desynchronise.

Raises:

Type	Description
ValueError	If a step names a transformer absent from the registry, or if the trial :class:FeatureEngineer rejects the chain.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def __init__(self, steps_config: list[dict[str, Any]], target_column: str):
    """Store the chain minus any splitter steps and validate it eagerly.

    Splitter steps are filtered out because they already ran upstream of the
    fold boundary. Unlike :class:`MergedBranchFoldAdapter`, row-changing
    steps are accepted here and ``changes_row_count`` merely reports the
    chain's nature: a single branch has no column-wise merge to
    desynchronise.

    Raises:
        ValueError: If a step names a transformer absent from the registry,
            or if the trial :class:`FeatureEngineer` rejects the chain.
    """
    self._steps_config = [
        step for step in steps_config if step.get("transformer") not in SPLITTER_STEP_TYPES
    ]
    self._target_column = target_column
    # True when any step reshapes the rows/target (resampling, row
    # drops); documents the chain's nature for callers that need it.
    self.changes_row_count = any(step_changes_row_count(step) for step in self._steps_config)
    # Validate eagerly (unknown transformer names, bad params) so a
    # misconfigured chain fails at construction, not mid-fold.
    # validate_preprocessing_steps does not check registry membership,
    # so resolve each step's calculator explicitly.
    FeatureEngineer(self._steps_config)
    for step in self._steps_config:
        NodeRegistry.get_calculator(step["transformer"])
    self._engineer: FeatureEngineer | None = None

fit_transform(X, y, *, sample_weight=None)

Fit a fresh engineer on this fold's (X, y) payload and return it.

A new :class:FeatureEngineer is built on every call, so nothing fitted on one fold survives into the next.

Raises:

Type	Description
ValueError	If X still embeds the configured target column.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
    """Fit a fresh engineer on this fold's ``(X, y)`` payload and return it.

    A new :class:`FeatureEngineer` is built on every call, so nothing fitted
    on one fold survives into the next.

    Raises:
        ValueError: If ``X`` still embeds the configured target column.
    """
    self._validate_payload(X)
    engineer = FeatureEngineer(self._steps_config)
    transformed, _metrics = engineer.fit_transform(
        (X, y), sample_weight=sample_weight, target_column=self._target_column
    )
    self._engineer = engineer
    self.train_sample_weight_ = engineer.train_sample_weight_
    return transformed

original_target_labels(labels)

Decode the recorded fitted target chain independently of row permutations.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def original_target_labels(self, labels: Any) -> Any:
    """Decode the recorded fitted target chain independently of row permutations."""
    if self._engineer is None:
        raise RuntimeError("Target decoding called before fit_transform()")
    return original_target_labels(self._engineer.fitted_steps, labels)

retain_split_metadata(column)

Drop retained split metadata after preprocessing, including during serving.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def retain_split_metadata(self, column: str) -> None:
    """Drop retained split metadata after preprocessing, including during serving."""
    step = {
        "name": f"exclude_split_metadata_{column}",
        "transformer": "DropMissingColumns",
        "params": {"columns": [column], "missing_threshold": None},
    }
    if step not in self._steps_config:
        self._steps_config = [*self._steps_config, step]

transform(X, y)

Apply the engineer fitted by the last :meth:fit_transform.

Always returns an (X, y) pair, re-pairing when an applier handed back a bare frame, so callers can rely on the FoldPreprocessor protocol shape.

Raises:

Type	Description
RuntimeError	If called before :meth:fit_transform.
ValueError	If X still embeds the configured target column.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
    """Apply the engineer fitted by the last :meth:`fit_transform`.

    Always returns an ``(X, y)`` pair, re-pairing when an applier handed
    back a bare frame, so callers can rely on the ``FoldPreprocessor``
    protocol shape.

    Raises:
        RuntimeError: If called before :meth:`fit_transform`.
        ValueError: If ``X`` still embeds the configured target column.
    """
    if self._engineer is None:
        raise RuntimeError("transform() called before fit_transform()")
    self._validate_payload(X)
    # At transform/inference time no step needs the target (target-aware
    # encoders use their fitted artifact; splitters/resampling are skipped).
    # Feed a bare frame when y is absent — a ``(X, None)`` tuple would trip
    # ``pack_pipeline_output``'s tuple-shape-lost diagnostic on every step.
    payload = (X, y) if y is not None else X
    transformed = self._engineer.transform(payload)
    self.last_transform_coverage_ = deepcopy(self._engineer.last_transform_coverage_)
    # Some appliers return a bare frame instead of the ``(X, y)`` payload;
    # re-pair so callers always get the FoldPreprocessor protocol shape.
    if not (isinstance(transformed, tuple) and len(transformed) == 2):
        transformed = (transformed, y)
    return transformed

transform_tracking_order(X)

Expose temporal row lineage for searcher prediction and nested thresholds.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def transform_tracking_order(self, X: Any) -> tuple[Any, Any]:
    """Expose temporal row lineage for searcher prediction and nested thresholds."""
    if self._engineer is None:
        raise RuntimeError("transform() called before fit_transform()")
    self._validate_payload(X)
    return self._engineer.transform_tracking_order(X)

FeatureGenerationApplier

Bases: BaseApplier

Append the columns described by a feature-generation artifact.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

class FeatureGenerationApplier(BaseApplier):
    """Append the columns described by a feature-generation artifact."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved operations, pinned similarity and learned group lookup shapes."""
        local_state_fields(
            raw, "feature_generation", {"type", "operations", "epsilon", "allow_overwrite"}
        )
        operations = raw["operations"]
        if type(operations) not in (list, tuple) or any(type(op) is not dict for op in operations):
            raise ValueError(
                "Fitted feature operations must be an ordered sequence of dictionaries."
            )
        if not isinstance(raw["allow_overwrite"], Decimal):
            local_scalar(raw["allow_overwrite"], "allow_overwrite")
        _validate_operation_types(operations)
        for op in operations:
            if op.get("operation_type") == "group_agg":
                _validate_group_mapping(op)
            if op.get("operation_type") == "similarity":
                _validate_similarity_backend(op.get("similarity_backend"))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved feature replay without granting arbitrary partition execution."""
        if engine not in {"pandas", "polars"}:
            return None
        FeatureGenerationApplier.validate_inference_state(state)
        contexts = {_operation_context(op, state["epsilon"], engine) for op in state["operations"]}
        if None in contexts:
            return None
        context = "global" if "global" in contexts else "row"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Evaluate each configured operation and append its output column to ``X``.

        Operations run in order, so a later one can read a column an earlier one
        created. Unsupported operation types raise ``ValueError`` before any
        operation runs. A failure inside a supported operation is logged and
        skipped, and an output name that already exists is suffixed until
        unique unless ``allow_overwrite`` says otherwise.

        Datetime extraction uses ``output_column`` exactly for one configured
        source and feature. With multiple features it produces ``name_feature``;
        with multiple sources it produces ``name_source_feature``. Without an
        explicit name, outputs remain ``source_feature``, optionally preceded
        by ``output_prefix``. Every datetime output follows the collision policy.
        Saved pipelines affected by formerly ignored names or overwritten
        columns require refitting after upgrading.

        Group aggregates require training-fitted mappings. Legacy artifacts
        lacking those mappings must be refitted before inference.
        """
        _validate_operation_types(params.get("operations", []))
        _check_similarity_runtime(params.get("operations", []))
        if any(
            op.get("operation_type") == "group_agg" and "group_agg_mapping" not in op
            for op in params.get("operations", [])
        ):
            raise ValueError(
                "FeatureGeneration group_agg requires fitted statistics; refit the node."
            )
        return apply_dual_engine(
            X, params, {"polars": _featgen_apply_polars, "pandas": _featgen_apply_pandas}
        )

apply(X, _y, params)

Evaluate each configured operation and append its output column to X.

Operations run in order, so a later one can read a column an earlier one created. Unsupported operation types raise ValueError before any operation runs. A failure inside a supported operation is logged and skipped, and an output name that already exists is suffixed until unique unless allow_overwrite says otherwise.

Datetime extraction uses output_column exactly for one configured source and feature. With multiple features it produces name_feature; with multiple sources it produces name_source_feature. Without an explicit name, outputs remain source_feature, optionally preceded by output_prefix. Every datetime output follows the collision policy. Saved pipelines affected by formerly ignored names or overwritten columns require refitting after upgrading.

Group aggregates require training-fitted mappings. Legacy artifacts lacking those mappings must be refitted before inference.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Evaluate each configured operation and append its output column to ``X``.

    Operations run in order, so a later one can read a column an earlier one
    created. Unsupported operation types raise ``ValueError`` before any
    operation runs. A failure inside a supported operation is logged and
    skipped, and an output name that already exists is suffixed until
    unique unless ``allow_overwrite`` says otherwise.

    Datetime extraction uses ``output_column`` exactly for one configured
    source and feature. With multiple features it produces ``name_feature``;
    with multiple sources it produces ``name_source_feature``. Without an
    explicit name, outputs remain ``source_feature``, optionally preceded
    by ``output_prefix``. Every datetime output follows the collision policy.
    Saved pipelines affected by formerly ignored names or overwritten
    columns require refitting after upgrading.

    Group aggregates require training-fitted mappings. Legacy artifacts
    lacking those mappings must be refitted before inference.
    """
    _validate_operation_types(params.get("operations", []))
    _check_similarity_runtime(params.get("operations", []))
    if any(
        op.get("operation_type") == "group_agg" and "group_agg_mapping" not in op
        for op in params.get("operations", [])
    ):
        raise ValueError(
            "FeatureGeneration group_agg requires fitted statistics; refit the node."
        )
    return apply_dual_engine(
        X, params, {"polars": _featgen_apply_polars, "pandas": _featgen_apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved feature replay without granting arbitrary partition execution.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved feature replay without granting arbitrary partition execution."""
    if engine not in {"pandas", "polars"}:
        return None
    FeatureGenerationApplier.validate_inference_state(state)
    contexts = {_operation_context(op, state["epsilon"], engine) for op in state["operations"]}
    if None in contexts:
        return None
    context = "global" if "global" in contexts else "row"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect saved operations, pinned similarity and learned group lookup shapes.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved operations, pinned similarity and learned group lookup shapes."""
    local_state_fields(
        raw, "feature_generation", {"type", "operations", "epsilon", "allow_overwrite"}
    )
    operations = raw["operations"]
    if type(operations) not in (list, tuple) or any(type(op) is not dict for op in operations):
        raise ValueError(
            "Fitted feature operations must be an ordered sequence of dictionaries."
        )
    if not isinstance(raw["allow_overwrite"], Decimal):
        local_scalar(raw["allow_overwrite"], "allow_overwrite")
    _validate_operation_types(operations)
    for op in operations:
        if op.get("operation_type") == "group_agg":
            _validate_group_mapping(op)
        if op.get("operation_type") == "similarity":
            _validate_similarity_backend(op.get("similarity_backend"))
    return raw

FeatureGenerationCalculator

Bases: BaseCalculator

Record row-local operations and fit group aggregates from training rows.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

@NodeRegistry.register("FeatureGeneration", FeatureGenerationApplier)
@NodeRegistry.register("FeatureMath", FeatureGenerationApplier)
@NodeRegistry.register("FeatureGenerationNode", FeatureGenerationApplier)
@node_meta(
    id="FeatureGenerationNode",
    name="Feature Generation (Math)",
    category="Feature Engineering",
    description="Generate new features using mathematical operations.",
    params={"operations": []},
    learns_from_data=True,
)
class FeatureGenerationCalculator(BaseCalculator):
    """Record row-local operations and fit group aggregates from training rows."""

    @fit_method
    def fit(
        self,
        X: Any,
        _y: Any,
        config: dict[str, Any],
    ) -> FeatureGenerationArtifact:  # pylint: disable=arguments-differ
        """Fit aggregate mappings against the intermediate training feature frame.

        Operations execute in order while fitting so an aggregate can consume
        an earlier generated column. Its saved mapping supplies held-out rows;
        unknown keys stay missing, and no inference values are aggregated.
        Configurations containing only row-local operations need no data fit.
        Unsupported operation types raise ``ValueError``; polynomial expansion
        belongs in the separate ``PolynomialFeatures`` node.
        """
        _validate_operation_types(config.get("operations", []))
        params: FeatureGenerationArtifact = {
            "type": "feature_generation",
            "operations": deepcopy(config.get("operations", [])),
            "epsilon": config.get("epsilon", DEFAULT_EPSILON),
            "allow_overwrite": config.get("allow_overwrite", False),
        }
        for op in params["operations"]:
            if op.get("operation_type") == "similarity":
                op["similarity_backend"] = _similarity_backend()
        if not any(op.get("operation_type") == "group_agg" for op in params["operations"]):
            return params

        working = X
        for index, op in enumerate(params["operations"]):
            if op.get("operation_type") == "group_agg":
                op["group_agg_mapping"] = _fit_group_aggregation(working, op)
            # apply_method exposes the public (data, params) call signature.
            working = FeatureGenerationApplier().apply(  # pylint: disable=no-value-for-parameter
                working, {**params, "operations": [op], "_operation_offset": index}
            )
        return params

fit(X, _y, config)

Fit aggregate mappings against the intermediate training feature frame.

Operations execute in order while fitting so an aggregate can consume an earlier generated column. Its saved mapping supplies held-out rows; unknown keys stay missing, and no inference values are aggregated. Configurations containing only row-local operations need no data fit. Unsupported operation types raise ValueError; polynomial expansion belongs in the separate PolynomialFeatures node.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/generation.py

@fit_method
def fit(
    self,
    X: Any,
    _y: Any,
    config: dict[str, Any],
) -> FeatureGenerationArtifact:  # pylint: disable=arguments-differ
    """Fit aggregate mappings against the intermediate training feature frame.

    Operations execute in order while fitting so an aggregate can consume
    an earlier generated column. Its saved mapping supplies held-out rows;
    unknown keys stay missing, and no inference values are aggregated.
    Configurations containing only row-local operations need no data fit.
    Unsupported operation types raise ``ValueError``; polynomial expansion
    belongs in the separate ``PolynomialFeatures`` node.
    """
    _validate_operation_types(config.get("operations", []))
    params: FeatureGenerationArtifact = {
        "type": "feature_generation",
        "operations": deepcopy(config.get("operations", [])),
        "epsilon": config.get("epsilon", DEFAULT_EPSILON),
        "allow_overwrite": config.get("allow_overwrite", False),
    }
    for op in params["operations"]:
        if op.get("operation_type") == "similarity":
            op["similarity_backend"] = _similarity_backend()
    if not any(op.get("operation_type") == "group_agg" for op in params["operations"]):
        return params

    working = X
    for index, op in enumerate(params["operations"]):
        if op.get("operation_type") == "group_agg":
            op["group_agg_mapping"] = _fit_group_aggregation(working, op)
        # apply_method exposes the public (data, params) call signature.
        working = FeatureGenerationApplier().apply(  # pylint: disable=no-value-for-parameter
            working, {**params, "operations": [op], "_operation_offset": index}
        )
    return params

FeatureInteractionApplier

Bases: BaseApplier

Append the multiplicative interaction columns named by the artifact.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

class FeatureInteractionApplier(BaseApplier):
    """Append the multiplicative interaction columns named by the artifact."""

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return interaction_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        return interaction_config(raw, state)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Compute each artifact combination as a column product and append it.

        A combination whose columns are absent from ``X`` is skipped rather than
        raising, so the node survives an upstream column drop; ``include_bias``
        adds a constant 1.0 ``interaction_bias`` column per existing row,
        including zero rows when no input columns remain. When no column is
        generated the frame is returned unchanged. Generated product names must
        be unique and must not collide with existing columns. An existing bias
        column retains its established passthrough behavior.
        """
        combos = [
            tuple(combo)
            for combo in params.get("combinations", [])
            if all(col in X.columns for col in combo)
        ]
        names = _build_interaction_feature_names(
            combos, params.get("include_bias", False) and _BIAS_COLUMN not in X.columns
        )
        _validate_generated_names(names, list(X.columns), "FeatureInteraction")
        return apply_dual_engine(
            X, params, {"polars": _interaction_apply_polars, "pandas": _interaction_apply_pandas}
        )

apply(X, _y, params)

Compute each artifact combination as a column product and append it.

A combination whose columns are absent from X is skipped rather than raising, so the node survives an upstream column drop; include_bias adds a constant 1.0 interaction_bias column per existing row, including zero rows when no input columns remain. When no column is generated the frame is returned unchanged. Generated product names must be unique and must not collide with existing columns. An existing bias column retains its established passthrough behavior.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Compute each artifact combination as a column product and append it.

    A combination whose columns are absent from ``X`` is skipped rather than
    raising, so the node survives an upstream column drop; ``include_bias``
    adds a constant 1.0 ``interaction_bias`` column per existing row,
    including zero rows when no input columns remain. When no column is
    generated the frame is returned unchanged. Generated product names must
    be unique and must not collide with existing columns. An existing bias
    column retains its established passthrough behavior.
    """
    combos = [
        tuple(combo)
        for combo in params.get("combinations", [])
        if all(col in X.columns for col in combo)
    ]
    names = _build_interaction_feature_names(
        combos, params.get("include_bias", False) and _BIAS_COLUMN not in X.columns
    )
    _validate_generated_names(names, list(X.columns), "FeatureInteraction")
    return apply_dual_engine(
        X, params, {"polars": _interaction_apply_polars, "pandas": _interaction_apply_pandas}
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    return interaction_config(raw, state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return interaction_state(raw)

FeatureInteractionCalculator

Bases: BaseCalculator

Resolve which cross-products to generate, without computing them.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

@NodeRegistry.register(
    "FeatureInteraction",
    FeatureInteractionApplier,
    execution_capabilities=(
        ExecutionCapability("pandas", "apply", "python_batch", "preserve", "row"),
        ExecutionCapability("polars", "apply", "local", "preserve", "row"),
    ),
)
@node_meta(
    id="FeatureInteraction",
    name="Feature Interaction",
    category="Feature Engineering",
    description=(
        "Generate 2-way/3-way/4-way multiplicative interaction features between "
        "numeric columns, using deterministic regularization-friendly names."
    ),
    params={"columns": [], **_DEFAULT_OPTIONS},
    learns_from_data=False,
)
class FeatureInteractionCalculator(BaseCalculator):
    """Resolve which cross-products to generate, without computing them."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> FeatureInteractionArtifact:  # pylint: disable=arguments-differ
        """Validate the requested columns and degree, then enumerate the combinations.

        Columns are sorted into the artifact so the generated names depend only
        on the set of inputs, never on the order they were configured in.
        With ``interaction_only=True``, fewer columns than ``degree`` yields
        no products. With ``False``, one column is sufficient for self-products.
        An empty selection yields no products; bias generation is independent.

        Raises:
            ValueError: If a configured column is missing or non-numeric, or if
                ``degree`` falls outside the supported ``(2, 3, 4)``. Generated
                product names must also be unique and absent from the input.
        """
        cols = list(config.get("columns", []))
        X_pd = select_then_to_pandas(X, cols)

        _validate_interaction_columns(X_pd, cols)

        degree = config.get("degree", _DEFAULT_OPTIONS["degree"])
        _validate_interaction_degree(degree)
        interaction_only = config.get("interaction_only", _DEFAULT_OPTIONS["interaction_only"])
        include_bias = config.get("include_bias", _DEFAULT_OPTIONS["include_bias"])

        combos = _resolve_combinations(cols, degree, interaction_only)
        feature_names = _build_interaction_feature_names(combos, include_bias)
        generated = [
            name for name in feature_names if name != _BIAS_COLUMN or name not in X.columns
        ]
        _validate_generated_names(generated, list(X.columns), "FeatureInteraction")

        return cast(
            FeatureInteractionArtifact,
            {
                "type": "feature_interaction",
                "columns": sorted(cols),
                "degree": degree,
                "interaction_only": interaction_only,
                "include_bias": include_bias,
                "combinations": [list(c) for c in combos],
                "feature_names": feature_names,
            },
        )

fit(X, _y, config)

Validate the requested columns and degree, then enumerate the combinations.

Columns are sorted into the artifact so the generated names depend only on the set of inputs, never on the order they were configured in. With interaction_only=True, fewer columns than degree yields no products. With False, one column is sufficient for self-products. An empty selection yields no products; bias generation is independent.

Raises:

Type	Description
ValueError	If a configured column is missing or non-numeric, or if degree falls outside the supported (2, 3, 4). Generated product names must also be unique and absent from the input.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/interaction.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> FeatureInteractionArtifact:  # pylint: disable=arguments-differ
    """Validate the requested columns and degree, then enumerate the combinations.

    Columns are sorted into the artifact so the generated names depend only
    on the set of inputs, never on the order they were configured in.
    With ``interaction_only=True``, fewer columns than ``degree`` yields
    no products. With ``False``, one column is sufficient for self-products.
    An empty selection yields no products; bias generation is independent.

    Raises:
        ValueError: If a configured column is missing or non-numeric, or if
            ``degree`` falls outside the supported ``(2, 3, 4)``. Generated
            product names must also be unique and absent from the input.
    """
    cols = list(config.get("columns", []))
    X_pd = select_then_to_pandas(X, cols)

    _validate_interaction_columns(X_pd, cols)

    degree = config.get("degree", _DEFAULT_OPTIONS["degree"])
    _validate_interaction_degree(degree)
    interaction_only = config.get("interaction_only", _DEFAULT_OPTIONS["interaction_only"])
    include_bias = config.get("include_bias", _DEFAULT_OPTIONS["include_bias"])

    combos = _resolve_combinations(cols, degree, interaction_only)
    feature_names = _build_interaction_feature_names(combos, include_bias)
    generated = [
        name for name in feature_names if name != _BIAS_COLUMN or name not in X.columns
    ]
    _validate_generated_names(generated, list(X.columns), "FeatureInteraction")

    return cast(
        FeatureInteractionArtifact,
        {
            "type": "feature_interaction",
            "columns": sorted(cols),
            "degree": degree,
            "interaction_only": interaction_only,
            "include_bias": include_bias,
            "combinations": [list(c) for c in combos],
            "feature_names": feature_names,
        },
    )

GeneralBinningApplier

Bases: BaseBinningApplier

Inspect fitted bin decisions while inheriting the existing shared apply.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

class GeneralBinningApplier(BaseBinningApplier):
    """Inspect fitted bin decisions while inheriting the existing shared apply."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved edges, optional labels and display choices without fitting."""
        return _validate_binning_inference_state(raw, allow_custom_labels=True)

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved bin application as local row-preserving inference."""
        if engine not in ("pandas", "polars"):
            return None
        GeneralBinningApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

inference_capability(state, *, engine) staticmethod

Describe saved bin application as local row-preserving inference.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved bin application as local row-preserving inference."""
    if engine not in ("pandas", "polars"):
        return None
    GeneralBinningApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved edges, optional labels and display choices without fitting.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved edges, optional labels and display choices without fitting."""
    return _validate_binning_inference_state(raw, allow_custom_labels=True)

GeneralBinningCalculator

Bases: BaseCalculator

Master calculator that handles mixed strategies and per-column overrides.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@NodeRegistry.register("GeneralBinning", GeneralBinningApplier)
@node_meta(
    id="GeneralBinning",
    name="General Binning",
    category="Preprocessing",
    description="Bin continuous data into intervals.",
    params={"n_bins": 5, "strategy": "uniform", "columns": []},
    learns_from_data=True,
)
class GeneralBinningCalculator(BaseCalculator):
    """Master calculator that handles mixed strategies and per-column overrides."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralBinningArtifact:  # pylint: disable=arguments-differ
        """Fit bin edges per column under each column's own strategy.

        An explicitly empty ``columns`` list short-circuits to an empty artifact
        (the user unchecked everything), whereas an absent one falls through to
        :func:`detect_numeric_columns`. Only the selected columns are converted
        to pandas, since the fits are ``pd.cut`` / ``pd.qcut`` / sklearn-bound.
        Columns whose strategy cannot be fit are logged and skipped rather than
        failing the node, so the artifact may cover fewer columns than requested.
        """
        if user_picked_no_columns(config):
            return cast(GeneralBinningArtifact, {})

        input_columns = list(X.columns)
        X, columns = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)

        defaults = {
            "default_n_bins": config.get("n_bins", 5),
            "n_bins": config.get("equal_width_bins", config.get("n_bins", 5)),
            "q_bins": config.get("equal_frequency_bins", config.get("n_bins", 5)),
            "duplicates": config.get("duplicates", "drop"),
        }

        valid_cols = [c for c in columns if c in X.columns]
        bin_edges_map: dict[str, list[float]] = {}
        custom_labels_map: dict[str, Any] = {}

        for col in valid_cols:
            _fit_one_column_into_maps(X, col, config, defaults, bin_edges_map, custom_labels_map)

        artifact: dict[str, Any] = {
            "type": "general_binning",
            "bin_edges": bin_edges_map,
            "custom_labels": custom_labels_map,
        }
        artifact.update(_passthrough_artifact_options(config))
        _validate_binning_artifact(input_columns, artifact)
        _store_range_labels(artifact)
        return cast(GeneralBinningArtifact, artifact)

fit(X, _y, config)

Fit bin edges per column under each column's own strategy.

An explicitly empty columns list short-circuits to an empty artifact (the user unchecked everything), whereas an absent one falls through to :func:detect_numeric_columns. Only the selected columns are converted to pandas, since the fits are pd.cut / pd.qcut / sklearn-bound. Columns whose strategy cannot be fit are logged and skipped rather than failing the node, so the artifact may cover fewer columns than requested.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralBinningArtifact:  # pylint: disable=arguments-differ
    """Fit bin edges per column under each column's own strategy.

    An explicitly empty ``columns`` list short-circuits to an empty artifact
    (the user unchecked everything), whereas an absent one falls through to
    :func:`detect_numeric_columns`. Only the selected columns are converted
    to pandas, since the fits are ``pd.cut`` / ``pd.qcut`` / sklearn-bound.
    Columns whose strategy cannot be fit are logged and skipped rather than
    failing the node, so the artifact may cover fewer columns than requested.
    """
    if user_picked_no_columns(config):
        return cast(GeneralBinningArtifact, {})

    input_columns = list(X.columns)
    X, columns = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)

    defaults = {
        "default_n_bins": config.get("n_bins", 5),
        "n_bins": config.get("equal_width_bins", config.get("n_bins", 5)),
        "q_bins": config.get("equal_frequency_bins", config.get("n_bins", 5)),
        "duplicates": config.get("duplicates", "drop"),
    }

    valid_cols = [c for c in columns if c in X.columns]
    bin_edges_map: dict[str, list[float]] = {}
    custom_labels_map: dict[str, Any] = {}

    for col in valid_cols:
        _fit_one_column_into_maps(X, col, config, defaults, bin_edges_map, custom_labels_map)

    artifact: dict[str, Any] = {
        "type": "general_binning",
        "bin_edges": bin_edges_map,
        "custom_labels": custom_labels_map,
    }
    artifact.update(_passthrough_artifact_options(config))
    _validate_binning_artifact(input_columns, artifact)
    _store_range_labels(artifact)
    return cast(GeneralBinningArtifact, artifact)

GeneralTransformationApplier

Bases: BaseApplier

Apply per-column simple ops or fitted Box-Cox/Yeo-Johnson transforms.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

class GeneralTransformationApplier(BaseApplier):
    """Apply per-column simple ops or fitted Box-Cox/Yeo-Johnson transforms."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect ordered saved rules while retaining the existing simple-rule checks."""
        local_state_fields(raw, "general_transformation", {"type", "transformations"})
        rules = raw["transformations"]
        if type(rules) not in (list, tuple):
            raise ValueError("GeneralTransformation rules must be a list or tuple.")
        for rule in rules:
            _validate_general_rule(rule)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Separate pointwise formulas from batch-dependent power failure fallback."""
        if engine not in ("pandas", "polars"):
            return None
        GeneralTransformationApplier.validate_inference_state(state)
        has_power = any(rule.get("method") in _POWER_METHODS for rule in state["transformations"])
        return ExecutionCapability(
            engine, "apply", "local", "preserve", "global" if has_power else "row"
        )

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Run each configured transformation on the active engine; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        transformations = params.get("transformations", [])
        if not transformations:
            return X, _y

        X_out = X
        for item in transformations:
            col = item.get("column")
            method = item.get("method")
            if col not in X_out.columns:
                continue
            if method in _POWER_METHODS:
                X_out = _apply_power_to_polars_col(X_out, item)
                continue
            op = _POLARS_OPS.get(method)
            if op is None:
                continue
            X_out = _apply_polars_op(X_out, item, op)
        return X_out, _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        transformations = params.get("transformations", [])
        if not transformations:
            return X, _y

        df_out = X.copy()
        for item in transformations:
            col = item.get("column")
            method = item.get("method")
            if col not in df_out.columns:
                continue
            if method in _POWER_METHODS:
                df_out = _apply_power_to_pandas_col(df_out, item)
                continue
            op = _PANDAS_OPS.get(method)
            if op is None:
                continue
            df_out[col] = op(pd.to_numeric(df_out[col], errors="coerce"), item)
        return df_out, _y

apply(X, _y, params)

Run each configured transformation on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Run each configured transformation on the active engine; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Separate pointwise formulas from batch-dependent power failure fallback.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Separate pointwise formulas from batch-dependent power failure fallback."""
    if engine not in ("pandas", "polars"):
        return None
    GeneralTransformationApplier.validate_inference_state(state)
    has_power = any(rule.get("method") in _POWER_METHODS for rule in state["transformations"])
    return ExecutionCapability(
        engine, "apply", "local", "preserve", "global" if has_power else "row"
    )

validate_inference_state(raw) staticmethod

Inspect ordered saved rules while retaining the existing simple-rule checks.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect ordered saved rules while retaining the existing simple-rule checks."""
    local_state_fields(raw, "general_transformation", {"type", "transformations"})
    rules = raw["transformations"]
    if type(rules) not in (list, tuple):
        raise ValueError("GeneralTransformation rules must be a list or tuple.")
    for rule in rules:
        _validate_general_rule(rule)
    return raw

GeneralTransformationCalculator

Bases: BaseCalculator

Resolve configured transformations, fitting lambdas for the power methods.

Rules run in list order. Each power rule is fitted on the training values produced by earlier rules on the same column, matching artifact replay. Each power rule accepts standardize (default True). Set it to False to apply only the fitted Box-Cox or Yeo-Johnson transform. The artifact retains the choice for subsequent transforms.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

@NodeRegistry.register("GeneralTransformation", GeneralTransformationApplier)
@node_meta(
    id="GeneralTransformation",
    name="General Transformation",
    category="Preprocessing",
    description="Apply various function transformations (log, sqrt, square, exp) to columns.",
    params={"transformations": []},
    learns_from_data=True,
)
class GeneralTransformationCalculator(BaseCalculator):
    """Resolve configured transformations, fitting lambdas for the power methods.

    Rules run in list order. Each power rule is fitted on the training values
    produced by earlier rules on the same column, matching artifact replay.
    Each power rule accepts ``standardize`` (default ``True``). Set it to
    ``False`` to apply only the fitted Box-Cox or Yeo-Johnson transform.
    The artifact retains the choice for subsequent transforms.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return updated dtypes for transformed numeric columns."""
        schema = input_schema
        touched: set[str] = set()
        for item in config.get("transformations", []):
            col = item.get("column")
            method = item.get("method")
            if col not in schema.columns or method is None:
                continue
            if method in _FLOAT_METHODS:
                touched.add(col)

        for col in touched:
            schema = schema.with_dtype(col, "float64")
        return schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralTransformationArtifact:  # pylint: disable=arguments-differ
        """Fit power lambdas in rule order; retain simple ops in the artifact.

        Box-Cox is skipped (with a warning) for non-positive values at that
        point in the rule sequence. A fit failure logs and skips that rule.
        """
        # Config: {'transformations': [{'column': 'col1', 'method': 'log'},
        #                              {'column': 'col2', 'method': 'yeo-johnson'}]}
        is_polars = get_engine(X).name == EngineName.POLARS
        fitted_transformations: list[dict[str, Any]] = []
        transformations = config.get("transformations", [])
        last_power_rule = {
            item.get("column"): index
            for index, item in enumerate(transformations)
            if item.get("method") in _POWER_METHODS
        }
        working_X = X
        applier = GeneralTransformationApplier()

        for index, item in enumerate(transformations):
            col = item.get("column")
            if col not in X.columns:
                continue

            fitted_item = _fit_transformation_rule(working_X, col, item, is_polars)
            if fitted_item is None:
                continue

            fitted_transformations.append(fitted_item)
            # Only materialize rules whose output is needed by a later fit.
            # Reuse replay semantics without mutating the caller's training data.
            if index < last_power_rule.get(col, -1):
                # apply_method exposes the public (data, params) call signature.
                working_X = applier.apply(  # pylint: disable=no-value-for-parameter
                    working_X, {"transformations": [fitted_item]}
                )

        return {
            "type": "general_transformation",
            "transformations": fitted_transformations,
        }

fit(X, _y, config)

Fit power lambdas in rule order; retain simple ops in the artifact.

Box-Cox is skipped (with a warning) for non-positive values at that point in the rule sequence. A fit failure logs and skips that rule.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeneralTransformationArtifact:  # pylint: disable=arguments-differ
    """Fit power lambdas in rule order; retain simple ops in the artifact.

    Box-Cox is skipped (with a warning) for non-positive values at that
    point in the rule sequence. A fit failure logs and skips that rule.
    """
    # Config: {'transformations': [{'column': 'col1', 'method': 'log'},
    #                              {'column': 'col2', 'method': 'yeo-johnson'}]}
    is_polars = get_engine(X).name == EngineName.POLARS
    fitted_transformations: list[dict[str, Any]] = []
    transformations = config.get("transformations", [])
    last_power_rule = {
        item.get("column"): index
        for index, item in enumerate(transformations)
        if item.get("method") in _POWER_METHODS
    }
    working_X = X
    applier = GeneralTransformationApplier()

    for index, item in enumerate(transformations):
        col = item.get("column")
        if col not in X.columns:
            continue

        fitted_item = _fit_transformation_rule(working_X, col, item, is_polars)
        if fitted_item is None:
            continue

        fitted_transformations.append(fitted_item)
        # Only materialize rules whose output is needed by a later fit.
        # Reuse replay semantics without mutating the caller's training data.
        if index < last_power_rule.get(col, -1):
            # apply_method exposes the public (data, params) call signature.
            working_X = applier.apply(  # pylint: disable=no-value-for-parameter
                working_X, {"transformations": [fitted_item]}
            )

    return {
        "type": "general_transformation",
        "transformations": fitted_transformations,
    }

infer_output_schema(input_schema, config)

Return updated dtypes for transformed numeric columns.

Source code in skyulf-core/skyulf/preprocessing/transformations/general.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return updated dtypes for transformed numeric columns."""
    schema = input_schema
    touched: set[str] = set()
    for item in config.get("transformations", []):
        col = item.get("column")
        method = item.get("method")
        if col not in schema.columns or method is None:
            continue
        if method in _FLOAT_METHODS:
            touched.add(col)

    for col in touched:
        schema = schema.with_dtype(col, "float64")
    return schema

GeoDistanceApplier

Bases: BaseApplier

Append the configured pairwise distance as a new numeric column.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

class GeoDistanceApplier(BaseApplier):
    """Append the configured pairwise distance as a new numeric column."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved coordinate names and distance choices without touching request data."""
        columns = {"lat1_col", "lon1_col", "lat2_col", "lon2_col", "output_column"}
        local_state_fields(raw, "geo_distance", {"type", "method", "unit", *columns})
        if any(type(raw[key]) is not str or not raw[key] for key in columns):
            raise ValueError("GeoDistance fitted column names must be nonempty strings.")
        _validate_geo_distance_method_unit(raw["method"], raw["unit"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe existing coordinate math as local row-preserving inference."""
        if engine not in ("pandas", "polars"):
            return None
        GeoDistanceApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Compute the distance column on the active engine; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": _geo_distance_apply_polars, "pandas": _geo_distance_apply_pandas}
        )

apply(X, _y, params)

Compute the distance column on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Compute the distance column on the active engine; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": _geo_distance_apply_polars, "pandas": _geo_distance_apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe existing coordinate math as local row-preserving inference.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe existing coordinate math as local row-preserving inference."""
    if engine not in ("pandas", "polars"):
        return None
    GeoDistanceApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved coordinate names and distance choices without touching request data.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved coordinate names and distance choices without touching request data."""
    columns = {"lat1_col", "lon1_col", "lat2_col", "lon2_col", "output_column"}
    local_state_fields(raw, "geo_distance", {"type", "method", "unit", *columns})
    if any(type(raw[key]) is not str or not raw[key] for key in columns):
        raise ValueError("GeoDistance fitted column names must be nonempty strings.")
    _validate_geo_distance_method_unit(raw["method"], raw["unit"])
    return raw

GeoDistanceCalculator

Bases: BaseCalculator

Validate the coordinate columns, method, and unit for the distance computation.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

@NodeRegistry.register("GeoDistance", GeoDistanceApplier)
@node_meta(
    id="GeoDistance",
    name="Geo Distance",
    category="Feature Engineering",
    description=(
        "Compute the great-circle (haversine) or flat-plane (euclidean) distance "
        "between two lat/lon coordinate pairs, appended as a new numeric column."
    ),
    params={
        "lat1_col": "",
        "lon1_col": "",
        "lat2_col": "",
        "lon2_col": "",
        "method": "haversine",
        "unit": "km",
        "output_column": "",
    },
    learns_from_data=False,
)
class GeoDistanceCalculator(BaseCalculator):
    """Validate the coordinate columns, method, and unit for the distance computation."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Expose the distance feature before execution when all coordinate inputs are known."""
        coordinates = (config.get(key) for key in ("lat1_col", "lon1_col", "lat2_col", "lon2_col"))
        if not all(column in input_schema for column in coordinates):
            return None
        _validate_geo_distance_method_unit(
            config.get("method", "haversine"), config.get("unit", "km")
        )
        output = _distance_output_column(config)
        return input_schema.add(output, "float64").with_dtype(output, "float64")

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeoDistanceArtifact:  # pylint: disable=arguments-differ
        """Validate the distance config and resolve its output column name.

        An omitted or empty ``output_column`` becomes ``geo_distance_km`` or
        ``geo_distance_mi`` according to ``unit``. Explicit names are retained
        in the artifact and reused during inference.
        """
        lat1_col = config.get("lat1_col", "")
        lon1_col = config.get("lon1_col", "")
        lat2_col = config.get("lat2_col", "")
        lon2_col = config.get("lon2_col", "")
        cols = [lat1_col, lon1_col, lat2_col, lon2_col]
        X_pd = select_then_to_pandas(X, cols)
        _validate_geo_distance_columns(X_pd, cols)

        method = config.get("method", "haversine")
        unit = config.get("unit", "km")
        _validate_geo_distance_method_unit(method, unit)

        output_column = _distance_output_column(config)

        return cast(
            GeoDistanceArtifact,
            {
                "type": "geo_distance",
                "lat1_col": lat1_col,
                "lon1_col": lon1_col,
                "lat2_col": lat2_col,
                "lon2_col": lon2_col,
                "method": method,
                "unit": unit,
                "output_column": output_column,
            },
        )

fit(X, _y, config)

Validate the distance config and resolve its output column name.

An omitted or empty output_column becomes geo_distance_km or geo_distance_mi according to unit. Explicit names are retained in the artifact and reused during inference.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GeoDistanceArtifact:  # pylint: disable=arguments-differ
    """Validate the distance config and resolve its output column name.

    An omitted or empty ``output_column`` becomes ``geo_distance_km`` or
    ``geo_distance_mi`` according to ``unit``. Explicit names are retained
    in the artifact and reused during inference.
    """
    lat1_col = config.get("lat1_col", "")
    lon1_col = config.get("lon1_col", "")
    lat2_col = config.get("lat2_col", "")
    lon2_col = config.get("lon2_col", "")
    cols = [lat1_col, lon1_col, lat2_col, lon2_col]
    X_pd = select_then_to_pandas(X, cols)
    _validate_geo_distance_columns(X_pd, cols)

    method = config.get("method", "haversine")
    unit = config.get("unit", "km")
    _validate_geo_distance_method_unit(method, unit)

    output_column = _distance_output_column(config)

    return cast(
        GeoDistanceArtifact,
        {
            "type": "geo_distance",
            "lat1_col": lat1_col,
            "lon1_col": lon1_col,
            "lat2_col": lat2_col,
            "lon2_col": lon2_col,
            "method": method,
            "unit": unit,
            "output_column": output_column,
        },
    )

infer_output_schema(input_schema, config)

Expose the distance feature before execution when all coordinate inputs are known.

Source code in skyulf-core/skyulf/preprocessing/geo/distance.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Expose the distance feature before execution when all coordinate inputs are known."""
    coordinates = (config.get(key) for key in ("lat1_col", "lon1_col", "lat2_col", "lon2_col"))
    if not all(column in input_schema for column in coordinates):
        return None
    _validate_geo_distance_method_unit(
        config.get("method", "haversine"), config.get("unit", "km")
    )
    output = _distance_output_column(config)
    return input_schema.add(output, "float64").with_dtype(output, "float64")

GroupImputerApplier

Bases: BaseApplier

Fill gaps with the training value of each row's group, then with the global value.

Rows whose group was unseen in training, whose group key is null, or whose group had no observed value fall back to the column's global training value. The scoring batch's own values are never used to compute fills.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

class GroupImputerApplier(BaseApplier):
    """Fill gaps with the training value of each row's group, then with the global value.

    Rows whose group was unseen in training, whose group key is null, or whose
    group had no observed value fall back to the column's global training
    value. The scoring batch's own values are never used to compute fills.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return _group_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        return _imputation_config("GroupImputer", fitted_columns(raw, state), state)

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Fill the learned columns on the active engine; ``y`` passes through."""
        if not params.get("columns"):
            return X if y is None else (X, y)
        _require_group_column(_native(X).columns, params["group_by"])
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        out = X.copy()
        keys = out[params["group_by"]].astype(object)
        numeric = params["strategy"] in _NUMERIC_STRATEGIES
        for column in params["columns"]:
            if column not in out.columns:
                continue
            series = out[column]
            group_fill = _pandas_group_fill(
                keys, series, params["group_values"][column], params["fill_values"].get(column)
            )
            if numeric:
                if not pd.api.types.is_float_dtype(series) or is_decimal_series(series):
                    series = pd.to_numeric(series).astype("float64")
                group_fill = pd.to_numeric(group_fill).astype("float64")
            series = _extend_categories(series, group_fill)
            out[column] = series.where(series.notna(), group_fill.to_numpy())
        return out, y

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        numeric = params["strategy"] in _NUMERIC_STRATEGIES
        keys = pl.col(params["group_by"])
        key_dtype = X.schema[params["group_by"]]
        if isinstance(key_dtype, (pl.Categorical, pl.Enum)):
            keys = keys.cast(pl.Utf8)
            key_dtype = pl.String
        exprs = [
            _polars_fill(X.schema[column], column, keys, key_dtype, params, numeric)
            for column in params["columns"]
            if column in X.columns
        ]
        return X.with_columns(exprs), y

apply(X, y, params)

Fill the learned columns on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Fill the learned columns on the active engine; ``y`` passes through."""
    if not params.get("columns"):
        return X if y is None else (X, y)
    _require_group_column(_native(X).columns, params["group_by"])
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    return _imputation_config("GroupImputer", fitted_columns(raw, state), state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return _group_state(raw)

GroupImputerCalculator

Bases: BaseCalculator

Learn per-group and global fill values from training rows.

group_by names the column that defines the groups. strategy is mean, median or most_frequent (mode is an alias). Leave columns empty to fill every numeric column (mean/median) or every column (most_frequent), excluding the group key and the target.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

@NodeRegistry.register(
    "GroupImputer",
    GroupImputerApplier,
    execution_capabilities=tuple(
        ExecutionCapability(
            engine,
            "apply",
            execution_kind,
            "preserve",
            "row",
            config_match=(("strategy", strategy),),
        )
        for engine, execution_kind in (("pandas", "python_batch"), ("polars", "local"))
        for strategy in ("mean", "most_frequent")
    )
    + tuple(
        ExecutionCapability(
            engine, "apply", "local", "preserve", "row", config_match=(("strategy", "median"),)
        )
        for engine in ("pandas", "polars")
    ),
)
@node_meta(
    id="GroupImputer",
    name="Group Imputer",
    category="Preprocessing",
    description=(
        "Fills missing values with the mean, median or most frequent value of each row's "
        "group (for example per industry), falling back to the overall value."
    ),
    params={"group_by": "", "strategy": "mean", "columns": []},
    learns_from_data=True,
)
class GroupImputerCalculator(BaseCalculator):
    """Learn per-group and global fill values from training rows.

    ``group_by`` names the column that defines the groups. ``strategy`` is
    ``mean``, ``median`` or ``most_frequent`` (``mode`` is an alias). Leave
    ``columns`` empty to fill every numeric column (mean/median) or every
    column (most_frequent), excluding the group key and the target.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Promote filled columns to ``float64`` for mean/median; keep the schema otherwise."""
        if config.get("strategy", "mean") not in _NUMERIC_STRATEGIES:
            return input_schema
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GroupImputerArtifact:  # pylint: disable=arguments-differ
        """Compute ``[group, value]`` pairs and a global fallback for every filled column."""
        group_by = config.get("group_by")
        if not group_by:
            raise ValueError(
                "GroupImputer needs group_by: the column whose groups get their own fill value."
            )
        strategy = _strategy(config)
        X = _native(X)
        _require_group_column(X.columns, group_by)
        columns = _fill_columns(X, config, group_by, strategy)
        frame = _training_frame(X, columns, group_by, strategy)
        fill_values = {}
        for column in columns:
            values = frame[column]
            if strategy in _NUMERIC_STRATEGIES:
                values = pd.to_numeric(values)
            fill_values[column] = _statistic(values, strategy)
        return {
            "type": "group_imputer",
            "group_by": group_by,
            "strategy": strategy,
            "columns": columns,
            "group_values": {c: _group_values(frame, c, group_by, strategy) for c in columns},
            "fill_values": fill_values,
        }

fit(X, _y, config)

Compute [group, value] pairs and a global fallback for every filled column.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> GroupImputerArtifact:  # pylint: disable=arguments-differ
    """Compute ``[group, value]`` pairs and a global fallback for every filled column."""
    group_by = config.get("group_by")
    if not group_by:
        raise ValueError(
            "GroupImputer needs group_by: the column whose groups get their own fill value."
        )
    strategy = _strategy(config)
    X = _native(X)
    _require_group_column(X.columns, group_by)
    columns = _fill_columns(X, config, group_by, strategy)
    frame = _training_frame(X, columns, group_by, strategy)
    fill_values = {}
    for column in columns:
        values = frame[column]
        if strategy in _NUMERIC_STRATEGIES:
            values = pd.to_numeric(values)
        fill_values[column] = _statistic(values, strategy)
    return {
        "type": "group_imputer",
        "group_by": group_by,
        "strategy": strategy,
        "columns": columns,
        "group_values": {c: _group_values(frame, c, group_by, strategy) for c in columns},
        "fill_values": fill_values,
    }

infer_output_schema(input_schema, config)

Promote filled columns to float64 for mean/median; keep the schema otherwise.

Source code in skyulf-core/skyulf/preprocessing/imputation/group.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Promote filled columns to ``float64`` for mean/median; keep the schema otherwise."""
    if config.get("strategy", "mean") not in _NUMERIC_STRATEGIES:
        return input_schema
    return promote_configured_columns_to_float64(input_schema, config)

H3IndexApplier

Bases: BaseApplier

Append each row's H3 hexagonal cell index as a new string column.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

class H3IndexApplier(BaseApplier):
    """Append each row's H3 hexagonal cell index as a new string column."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved coordinate choices without importing H3 or computing cells."""
        local_state_fields(
            raw, "h3_index", {"type", "lat_col", "lon_col", "resolution", "output_column"}
        )
        if any(not isinstance(raw[key], str) or not raw[key] for key in ("lat_col", "lon_col")):
            raise ValueError("H3Index fitted coordinate names must be nonempty strings.")
        if not isinstance(raw["output_column"], Hashable):
            raise ValueError("H3Index fitted output name must be hashable.")
        _validate_h3_resolution(raw["resolution"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved cell indexing while retaining the optional runtime dependency."""
        if engine not in ("pandas", "polars"):
            return None
        H3IndexApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Compute the per-row cell indices on the active engine; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": _h3_index_apply_polars, "pandas": _h3_index_apply_pandas}
        )

apply(X, _y, params)

Compute the per-row cell indices on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Compute the per-row cell indices on the active engine; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": _h3_index_apply_polars, "pandas": _h3_index_apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved cell indexing while retaining the optional runtime dependency.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved cell indexing while retaining the optional runtime dependency."""
    if engine not in ("pandas", "polars"):
        return None
    H3IndexApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved coordinate choices without importing H3 or computing cells.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved coordinate choices without importing H3 or computing cells."""
    local_state_fields(
        raw, "h3_index", {"type", "lat_col", "lon_col", "resolution", "output_column"}
    )
    if any(not isinstance(raw[key], str) or not raw[key] for key in ("lat_col", "lon_col")):
        raise ValueError("H3Index fitted coordinate names must be nonempty strings.")
    if not isinstance(raw["output_column"], Hashable):
        raise ValueError("H3Index fitted output name must be hashable.")
    _validate_h3_resolution(raw["resolution"])
    return raw

H3IndexCalculator

Bases: BaseCalculator

Validate the lat/lon columns and resolution for H3 cell indexing.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

@NodeRegistry.register("H3Index", H3IndexApplier)
@node_meta(
    id="H3Index",
    name="H3 Index",
    category="Feature Engineering",
    description=(
        "Compute the Uber H3 hierarchical hexagonal grid cell index for each "
        "lat/lon pair, appended as a new string column. Requires the optional "
        "'h3' package (pip install skyulf-core[geo])."
    ),
    params={"lat_col": "", "lon_col": "", "resolution": 9, "output_column": "h3_index"},
    learns_from_data=False,
)
class H3IndexCalculator(BaseCalculator):
    """Validate the lat/lon columns and resolution for H3 cell indexing."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> H3IndexArtifact:  # pylint: disable=arguments-differ
        """Import ``h3`` eagerly to fail fast, then validate the columns and resolution."""
        # Fail fast with a clear message if the optional dependency is missing,
        # rather than only failing later inside Applier.apply().
        _import_h3()

        lat_col = config.get("lat_col", "")
        lon_col = config.get("lon_col", "")
        X_pd = select_then_to_pandas(X, [lat_col, lon_col])
        _validate_h3_columns(X_pd, lat_col, lon_col)

        resolution = config.get("resolution", 9)
        _validate_h3_resolution(resolution)

        output_column = config.get("output_column", "h3_index")

        return cast(
            H3IndexArtifact,
            {
                "type": "h3_index",
                "lat_col": lat_col,
                "lon_col": lon_col,
                "resolution": resolution,
                "output_column": output_column,
            },
        )

fit(X, _y, config)

Import h3 eagerly to fail fast, then validate the columns and resolution.

Source code in skyulf-core/skyulf/preprocessing/geo/h3_index.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> H3IndexArtifact:  # pylint: disable=arguments-differ
    """Import ``h3`` eagerly to fail fast, then validate the columns and resolution."""
    # Fail fast with a clear message if the optional dependency is missing,
    # rather than only failing later inside Applier.apply().
    _import_h3()

    lat_col = config.get("lat_col", "")
    lon_col = config.get("lon_col", "")
    X_pd = select_then_to_pandas(X, [lat_col, lon_col])
    _validate_h3_columns(X_pd, lat_col, lon_col)

    resolution = config.get("resolution", 9)
    _validate_h3_resolution(resolution)

    output_column = config.get("output_column", "h3_index")

    return cast(
        H3IndexArtifact,
        {
            "type": "h3_index",
            "lat_col": lat_col,
            "lon_col": lon_col,
            "resolution": resolution,
            "output_column": output_column,
        },
    )

HashingVectorizerApplier

Bases: BaseApplier

Attach n_features hashed-token columns to the frame.

No vocabulary is consulted — tokens hash straight to column indices, so the same artifact works on text never seen at fit time and the output width is fixed by config. The trade-off is collisions, which the hashing trick accepts by design. The configured text columns are joined first, the dense result is concatenated on, and the sources survive unless drop_original is set. The polars path runs natively and falls back to a pandas round-trip when a text column is not String dtype.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

class HashingVectorizerApplier(BaseApplier):
    """Attach ``n_features`` hashed-token columns to the frame.

    No vocabulary is consulted — tokens hash straight to column indices, so the
    same artifact works on text never seen at fit time and the output width is
    fixed by config. The trade-off is collisions, which the hashing trick
    accepts by design. The configured text columns are joined first, the dense
    result is concatenated on, and the sources survive unless ``drop_original``
    is set. The polars path runs natively and falls back to a pandas round-trip
    when a text column is not String dtype.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect fixed dense width without hashing tokens or allocating the output."""
        vectorizer = _validate_local_vectorizer(
            raw, "hashing_vectorizer", HashingVectorizer, {"n_features", "norm"}
        )
        if vectorizer is not None:
            width = raw["n_features"]
            if not isinstance(width, Integral) or width != vectorizer.n_features:
                raise ValueError("Fitted hashing width disagrees with the saved vectorizer.")
            if len(raw["output_columns"]) != max(0, width):
                raise ValueError("Fitted hashing output names have the wrong width.")
            if raw["norm"] != vectorizer.norm:
                raise ValueError("Fitted hashing norm disagrees with the saved vectorizer.")
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe stateless document hashing while retaining native callback limits."""
        if engine not in ("pandas", "polars"):
            return None
        HashingVectorizerApplier.validate_inference_state(state)
        if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
        return apply_text_dual_engine(
            X, params, _hash_apply_pandas, _sklearn_vectorizer_apply_polars
        )

apply(X, _y, params)

Dispatch through the text-specific dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
    return apply_text_dual_engine(
        X, params, _hash_apply_pandas, _sklearn_vectorizer_apply_polars
    )

inference_capability(state, *, engine) staticmethod

Describe stateless document hashing while retaining native callback limits.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe stateless document hashing while retaining native callback limits."""
    if engine not in ("pandas", "polars"):
        return None
    HashingVectorizerApplier.validate_inference_state(state)
    if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect fixed dense width without hashing tokens or allocating the output.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect fixed dense width without hashing tokens or allocating the output."""
    vectorizer = _validate_local_vectorizer(
        raw, "hashing_vectorizer", HashingVectorizer, {"n_features", "norm"}
    )
    if vectorizer is not None:
        width = raw["n_features"]
        if not isinstance(width, Integral) or width != vectorizer.n_features:
            raise ValueError("Fitted hashing width disagrees with the saved vectorizer.")
        if len(raw["output_columns"]) != max(0, width):
            raise ValueError("Fitted hashing output names have the wrong width.")
        if raw["norm"] != vectorizer.norm:
            raise ValueError("Fitted hashing norm disagrees with the saved vectorizer.")
    return raw

HashingVectorizerCalculator

Bases: BaseCalculator

Record the column list and build an unfitted HashingVectorizer from config.

Nothing is learned from the data (learns_from_data=False), so the artifact is fully reproducible from config and identical across runs — the property that makes it usable on streaming or very large corpora where building a vocabulary would not fit in memory.

Normalization defaults to "l2"; "l1" selects L1 normalization. The Canvas value "none" and Python None both disable normalization.

Schema inference conservatively inherits the base None result. A fixed n_features does not describe the whole output: fitting resolves source columns and excludes the target, including a separately supplied y. Runtime introspection determines the retained and generated columns.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

@NodeRegistry.register("hashing_vectorizer", HashingVectorizerApplier)
@node_meta(
    id="hashing_vectorizer",
    name="Hashing Vectorizer",
    category="Text",
    description=(
        "Stateless text vectorizer using the hashing trick. "
        "No vocabulary is stored — tokens are hashed directly to column indices. "
        "Suitable for very large or streaming datasets."
    ),
    params={
        "columns": [],
        "n_features": _DEFAULT_N_FEATURES,
        "norm": "l2",
        "alternate_sign": True,
        "lowercase": True,
        "stop_words": None,
        "drop_original": False,
    },
    tags=["text", "nlp", "hashing", "vectorizer", "stateless"],
    learns_from_data=False,
)
class HashingVectorizerCalculator(BaseCalculator):
    """Record the column list and build an unfitted ``HashingVectorizer`` from config.

    Nothing is learned from the data (``learns_from_data=False``), so the artifact
    is fully reproducible from config and identical across runs — the property that
    makes it usable on streaming or very large corpora where building a vocabulary
    would not fit in memory.

    Normalization defaults to ``"l2"``; ``"l1"`` selects L1 normalization.
    The Canvas value ``"none"`` and Python ``None`` both disable normalization.

    Schema inference conservatively inherits the base ``None`` result. A fixed
    ``n_features`` does not describe the whole output: fitting resolves source
    columns and excludes the target, including a separately supplied ``y``.
    Runtime introspection determines the retained and generated columns.
    """

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> HashingVectorizerArtifact:  # pylint: disable=arguments-differ
        """Resolve the configured text columns and build the artifact without touching values.

        Uses the column-names-only resolver rather than the data-narrowing sibling,
        so a polars input is never converted to pandas just to fit. An empty or
        wholly-absent selection yields an empty artifact so the applier no-ops.
        """
        valid_cols = resolve_fit_text_valid_columns(X, config, _y)
        if valid_cols is None:
            return {}

        artifact = _build_hashing_artifact(config, valid_cols)
        validate_text_output_names(X, artifact, valid_cols)
        return artifact

fit(X, _y, config)

Resolve the configured text columns and build the artifact without touching values.

Uses the column-names-only resolver rather than the data-narrowing sibling, so a polars input is never converted to pandas just to fit. An empty or wholly-absent selection yields an empty artifact so the applier no-ops.

Source code in skyulf-core/skyulf/preprocessing/vectorization/hashing_vectorizer.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> HashingVectorizerArtifact:  # pylint: disable=arguments-differ
    """Resolve the configured text columns and build the artifact without touching values.

    Uses the column-names-only resolver rather than the data-narrowing sibling,
    so a polars input is never converted to pandas just to fit. An empty or
    wholly-absent selection yields an empty artifact so the applier no-ops.
    """
    valid_cols = resolve_fit_text_valid_columns(X, config, _y)
    if valid_cols is None:
        return {}

    artifact = _build_hashing_artifact(config, valid_cols)
    validate_text_output_names(X, artifact, valid_cols)
    return artifact

IQRApplier

Bases: BaseApplier

Drop rows outside the fitted per-column IQR bounds; null/NaN values are kept.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

class IQRApplier(BaseApplier):
    """Drop rows outside the fitted per-column IQR bounds; null/NaN values are kept."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned quartile limits without recomputing request statistics."""
        if local_state_fields(
            raw, "iqr", {"type", "bounds", "multiplier", "warnings"}, allow_empty=True
        ):
            validate_fitted_bounds(raw["bounds"], partial=False)
            validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved row filtering; prediction still rejects any removed rows."""
        if engine not in ("pandas", "polars"):
            return None
        IQRApplier.validate_inference_state(state)
        effect = "filter" if state.get("bounds") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted per-column bounds on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        mask = pl.repeat(True, pl.len())
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            values = pl.col(col).cast(pl.Float64, strict=False)
            col_mask = (values >= bound["lower"]) & (values <= bound["upper"])
            missing = values.is_null() | values.is_nan()
            mask = mask & (col_mask | missing)

        mask_series = X.select(mask.alias("mask")).get_column("mask")
        return X.filter(mask_series), _filter_y_polars(y, mask_series)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        mask = pd.Series(True, index=X.index)
        for col, bound in bounds.items():
            if col not in X.columns:
                continue
            series = pd.to_numeric(X[col], errors="coerce")
            col_mask = (series >= bound["lower"]) & (series <= bound["upper"])
            mask = mask & (col_mask | series.isna())

        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted per-column bounds on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted per-column bounds on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved row filtering; prediction still rejects any removed rows.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved row filtering; prediction still rejects any removed rows."""
    if engine not in ("pandas", "polars"):
        return None
    IQRApplier.validate_inference_state(state)
    effect = "filter" if state.get("bounds") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect learned quartile limits without recomputing request statistics.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned quartile limits without recomputing request statistics."""
    if local_state_fields(
        raw, "iqr", {"type", "bounds", "multiplier", "warnings"}, allow_empty=True
    ):
        validate_fitted_bounds(raw["bounds"], partial=False)
        validate_detector_warnings(raw["warnings"])
    return raw

IQRCalculator

Bases: BaseCalculator

Fit per-column bounds at the quartiles widened by multiplier times the IQR.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

@NodeRegistry.register("IQR", IQRApplier)
@node_meta(
    id="IQR",
    name="IQR Outlier Removal",
    category="Preprocessing",
    description="Remove outliers using Interquartile Range.",
    params={"multiplier": 1.5, "columns": []},
    learns_from_data=True,
)
class IQRCalculator(BaseCalculator):
    """Fit per-column bounds at the quartiles widened by ``multiplier`` times the IQR."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # IQR removes outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IQRArtifact:  # pylint: disable=arguments-differ
        """Compute quartile bounds per numeric column; warn on empty or non-numeric ones."""
        if user_picked_no_columns(config):
            return {}

        multiplier = config.get("multiplier", 1.5)
        # TODO(pandas-removal): stays on resolve_columns_then_to_pandas (not
        # ..._to_numpy) because `.quantile()` needs Pandas' linear
        # interpolation semantics, which differ from Polars' default
        # "nearest" interpolation (see _helpers.py note). Switching to numpy
        # would require doing the quantile math with
        # np.percentile(..., method="linear") plus matching Pandas'
        # errors="coerce" NaN-drop behavior before benchmarking equivalence.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        bounds: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            if series.empty:
                warnings.append(f"Column '{col}': Empty or non-numeric")
                continue
            q1 = series.quantile(0.25)
            q3 = series.quantile(0.75)
            iqr = q3 - q1
            bounds[col] = {"lower": q1 - (multiplier * iqr), "upper": q3 + (multiplier * iqr)}

        return {
            "type": "iqr",
            "bounds": bounds,
            "multiplier": multiplier,
            "warnings": warnings,
        }

fit(X, _y, config)

Compute quartile bounds per numeric column; warn on empty or non-numeric ones.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IQRArtifact:  # pylint: disable=arguments-differ
    """Compute quartile bounds per numeric column; warn on empty or non-numeric ones."""
    if user_picked_no_columns(config):
        return {}

    multiplier = config.get("multiplier", 1.5)
    # TODO(pandas-removal): stays on resolve_columns_then_to_pandas (not
    # ..._to_numpy) because `.quantile()` needs Pandas' linear
    # interpolation semantics, which differ from Polars' default
    # "nearest" interpolation (see _helpers.py note). Switching to numpy
    # would require doing the quantile math with
    # np.percentile(..., method="linear") plus matching Pandas'
    # errors="coerce" NaN-drop behavior before benchmarking equivalence.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    bounds: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        if series.empty:
            warnings.append(f"Column '{col}': Empty or non-numeric")
            continue
        q1 = series.quantile(0.25)
        q3 = series.quantile(0.75)
        iqr = q3 - q1
        bounds[col] = {"lower": q1 - (multiplier * iqr), "upper": q3 + (multiplier * iqr)}

    return {
        "type": "iqr",
        "bounds": bounds,
        "multiplier": multiplier,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/iqr.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # IQR removes outlier *rows*; column set is preserved.
    return input_schema

IterativeImputerApplier

Bases: BaseApplier

Fill missing values from the fitted MICE (chained-equations) imputer.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

class IterativeImputerApplier(BaseApplier):
    """Fill missing values from the fitted MICE (chained-equations) imputer."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned initial values and estimator sequence without executing them."""
        fields = {"type", "imputer_object", "columns", "estimator"}
        if local_state_fields(raw, "iterative_imputer", fields, allow_empty=True):
            imputer = _validate_local_imputer(raw, IterativeImputer)
            width = len(raw["columns"])
            _validate_iterative_initial(imputer, width)
            _validate_iterative_sequence(imputer, width)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Retain the all-missing request shortcut and abstain from stochastic replay."""
        if engine not in {"pandas", "polars"}:
            return None
        IterativeImputerApplier.validate_inference_state(state)
        context = "row"
        if state:
            imputer = state["imputer_object"]
            if imputer.sample_posterior or any(
                type(step.estimator) not in _LOCAL_ESTIMATORS
                for step in imputer.imputation_sequence_
            ):
                return None
            if imputer.n_iter_:
                context = "global"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Impute ``X`` with the stored sklearn imputer; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        imputer = params.get("imputer_object")
        if not resolve_valid_columns(X, cols) or not imputer:
            return X, _y
        return _sklearn_transform_subset(X, cols, imputer, is_polars=True), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        imputer = params.get("imputer_object")
        if not resolve_valid_columns(X, cols) or not imputer:
            return X, _y
        return _sklearn_transform_subset(X, cols, imputer, is_polars=False), _y

apply(X, _y, params)

Impute X with the stored sklearn imputer; y passes through.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Impute ``X`` with the stored sklearn imputer; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Retain the all-missing request shortcut and abstain from stochastic replay.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Retain the all-missing request shortcut and abstain from stochastic replay."""
    if engine not in {"pandas", "polars"}:
        return None
    IterativeImputerApplier.validate_inference_state(state)
    context = "row"
    if state:
        imputer = state["imputer_object"]
        if imputer.sample_posterior or any(
            type(step.estimator) not in _LOCAL_ESTIMATORS
            for step in imputer.imputation_sequence_
        ):
            return None
        if imputer.n_iter_:
            context = "global"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect learned initial values and estimator sequence without executing them.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned initial values and estimator sequence without executing them."""
    fields = {"type", "imputer_object", "columns", "estimator"}
    if local_state_fields(raw, "iterative_imputer", fields, allow_empty=True):
        imputer = _validate_local_imputer(raw, IterativeImputer)
        width = len(raw["columns"])
        _validate_iterative_initial(imputer, width)
        _validate_iterative_sequence(imputer, width)
    return raw

IterativeImputerCalculator

Bases: BaseCalculator

Fit a sklearn IterativeImputer with the configured estimator.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

@NodeRegistry.register("IterativeImputer", IterativeImputerApplier)
@node_meta(
    id="IterativeImputer",
    name="Iterative Imputer (MICE)",
    category="Preprocessing",
    description="Multivariate imputation using chained equations.",
    params={"max_iter": 10, "random_state": 0, "estimator": "BayesianRidge", "columns": []},
    learns_from_data=True,
)
class IterativeImputerCalculator(BaseCalculator):
    """Fit a sklearn ``IterativeImputer`` with the configured estimator."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with selected columns promoted to ``float64``."""
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IterativeImputerArtifact:  # pylint: disable=arguments-differ
        """Fit chained-equations imputation, dropping all-missing columns sklearn cannot impute."""
        if user_picked_no_columns(config):
            return {}

        max_iter = config.get("max_iter", 10)
        estimator_name = config.get("estimator", "BayesianRidge")
        random_state = config.get("random_state", 0)

        # KNN/Iterative imputers always fit through numpy — engine choice
        # doesn't affect the fit math, so we skip the Pandas hop entirely.
        X_np, cols = resolve_columns_then_to_numpy(X, config, detect_numeric_columns)
        if not cols:
            return {}

        # sklearn silently drops all-missing columns from transform() output,
        # which desyncs the artifact's column list from the imputer's width.
        X_np, cols = drop_all_missing_columns(X_np, cols, "IterativeImputerCalculator")
        if not cols:
            return {}

        estimator = _build_iterative_estimator(estimator_name)
        imputer = IterativeImputer(
            estimator=estimator, max_iter=max_iter, random_state=random_state
        )
        imputer.fit(X_np)

        return {
            "type": "iterative_imputer",
            "imputer_object": imputer,  # Not JSON serializable
            "columns": cols,
            "estimator": estimator_name,
        }

fit(X, _y, config)

Fit chained-equations imputation, dropping all-missing columns sklearn cannot impute.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> IterativeImputerArtifact:  # pylint: disable=arguments-differ
    """Fit chained-equations imputation, dropping all-missing columns sklearn cannot impute."""
    if user_picked_no_columns(config):
        return {}

    max_iter = config.get("max_iter", 10)
    estimator_name = config.get("estimator", "BayesianRidge")
    random_state = config.get("random_state", 0)

    # KNN/Iterative imputers always fit through numpy — engine choice
    # doesn't affect the fit math, so we skip the Pandas hop entirely.
    X_np, cols = resolve_columns_then_to_numpy(X, config, detect_numeric_columns)
    if not cols:
        return {}

    # sklearn silently drops all-missing columns from transform() output,
    # which desyncs the artifact's column list from the imputer's width.
    X_np, cols = drop_all_missing_columns(X_np, cols, "IterativeImputerCalculator")
    if not cols:
        return {}

    estimator = _build_iterative_estimator(estimator_name)
    imputer = IterativeImputer(
        estimator=estimator, max_iter=max_iter, random_state=random_state
    )
    imputer.fit(X_np)

    return {
        "type": "iterative_imputer",
        "imputer_object": imputer,  # Not JSON serializable
        "columns": cols,
        "estimator": estimator_name,
    }

infer_output_schema(input_schema, config)

Return a schema with selected columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/imputation/iterative.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with selected columns promoted to ``float64``."""
    return promote_configured_columns_to_float64(input_schema, config)

KBinsDiscretizerApplier

Bases: GeneralBinningApplier

Inspect sklearn-selected edges while reusing the shared cut implementation.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

class KBinsDiscretizerApplier(GeneralBinningApplier):
    """Inspect sklearn-selected edges while reusing the shared cut implementation."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the same saved artifact produced by GeneralBinning's sklearn path."""
        return GeneralBinningApplier.validate_inference_state(raw)

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe fixed learned bins without granting distributed execution."""
        if engine not in ("pandas", "polars"):
            return None
        KBinsDiscretizerApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

inference_capability(state, *, engine) staticmethod

Describe fixed learned bins without granting distributed execution.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe fixed learned bins without granting distributed execution."""
    if engine not in ("pandas", "polars"):
        return None
    KBinsDiscretizerApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect the same saved artifact produced by GeneralBinning's sklearn path.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the same saved artifact produced by GeneralBinning's sklearn path."""
    return GeneralBinningApplier.validate_inference_state(raw)

KBinsDiscretizerCalculator

Bases: GeneralBinningCalculator

Thin wrapper around :class:GeneralBinningCalculator with kbins strategy.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

@NodeRegistry.register("KBinsDiscretizer", KBinsDiscretizerApplier)
@node_meta(
    id="KBinsDiscretizer",
    name="K-Bins Discretizer",
    category="Preprocessing",
    description="Bin continuous data into intervals using sklearn KBinsDiscretizer.",
    params={"n_bins": 5, "encode": "ordinal", "strategy": "quantile", "columns": []},
    learns_from_data=True,
)
class KBinsDiscretizerCalculator(GeneralBinningCalculator):
    """Thin wrapper around :class:`GeneralBinningCalculator` with ``kbins`` strategy."""

    def fit(  # pylint: disable=arguments-differ
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        config: dict[str, Any],
    ) -> GeneralBinningArtifact:
        """Translate this node's public params onto GeneralBinning's internal names, then delegate.

        On a copy of *config* (the caller's dict is never mutated) it forces
        ``strategy="kbins"``, moves ``n_bins`` to ``kbins_n_bins``, and preserves
        a user-chosen non-``kbins`` ``strategy`` as ``kbins_strategy`` so
        ``quantile`` / ``uniform`` / ``kmeans`` still reach sklearn. Written with
        the raw ``(df, config)`` signature rather than ``@fit_method`` because it
        only rewrites config and forwards — the parent's decorated ``fit`` does
        the ``(X, y)`` unpack.
        """
        new_config = config.copy()
        new_config["strategy"] = "kbins"
        if "n_bins" in config:
            new_config["kbins_n_bins"] = config["n_bins"]
        if "strategy" in config and config["strategy"] != "kbins":
            new_config["kbins_strategy"] = config["strategy"]
        return super().fit(df, new_config)  # pylint: disable=no-value-for-parameter

fit(df, config)

Translate this node's public params onto GeneralBinning's internal names, then delegate.

On a copy of config (the caller's dict is never mutated) it forces strategy="kbins", moves n_bins to kbins_n_bins, and preserves a user-chosen non-kbins strategy as kbins_strategy so quantile / uniform / kmeans still reach sklearn. Written with the raw (df, config) signature rather than @fit_method because it only rewrites config and forwards — the parent's decorated fit does the (X, y) unpack.

Source code in skyulf-core/skyulf/preprocessing/bucketing.py

def fit(  # pylint: disable=arguments-differ
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    config: dict[str, Any],
) -> GeneralBinningArtifact:
    """Translate this node's public params onto GeneralBinning's internal names, then delegate.

    On a copy of *config* (the caller's dict is never mutated) it forces
    ``strategy="kbins"``, moves ``n_bins`` to ``kbins_n_bins``, and preserves
    a user-chosen non-``kbins`` ``strategy`` as ``kbins_strategy`` so
    ``quantile`` / ``uniform`` / ``kmeans`` still reach sklearn. Written with
    the raw ``(df, config)`` signature rather than ``@fit_method`` because it
    only rewrites config and forwards — the parent's decorated ``fit`` does
    the ``(X, y)`` unpack.
    """
    new_config = config.copy()
    new_config["strategy"] = "kbins"
    if "n_bins" in config:
        new_config["kbins_n_bins"] = config["n_bins"]
    if "strategy" in config and config["strategy"] != "kbins":
        new_config["kbins_strategy"] = config["strategy"]
    return super().fit(df, new_config)  # pylint: disable=no-value-for-parameter

KNNImputerApplier

Bases: BaseApplier

Fill missing values from the fitted k-nearest-neighbors imputer.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

class KNNImputerApplier(BaseApplier):
    """Fill missing values from the fitted k-nearest-neighbors imputer."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved neighbor estimator without refitting or normalizing it."""
        fields = {"type", "imputer_object", "columns", "n_neighbors", "weights"}
        if local_state_fields(raw, "knn_imputer", fields, allow_empty=True):
            imputer = _validate_local_imputer(raw, KNNImputer)
            _validate_knn_neighbors(imputer, len(raw["columns"]))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe native neighbor replay; arbitrary metrics or weights stay undeclared."""
        if engine not in {"pandas", "polars"}:
            return None
        KNNImputerApplier.validate_inference_state(state)
        if state:
            imputer = state["imputer_object"]
            if callable(imputer.weights) or callable(imputer.metric):
                return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Impute ``X`` with the stored sklearn imputer; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        imputer = params.get("imputer_object")
        if not resolve_valid_columns(X, cols) or not imputer:
            return X, _y
        return _sklearn_transform_subset(X, cols, imputer, is_polars=True), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        imputer = params.get("imputer_object")
        if not resolve_valid_columns(X, cols) or not imputer:
            return X, _y
        return _sklearn_transform_subset(X, cols, imputer, is_polars=False), _y

apply(X, _y, params)

Impute X with the stored sklearn imputer; y passes through.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Impute ``X`` with the stored sklearn imputer; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe native neighbor replay; arbitrary metrics or weights stay undeclared.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe native neighbor replay; arbitrary metrics or weights stay undeclared."""
    if engine not in {"pandas", "polars"}:
        return None
    KNNImputerApplier.validate_inference_state(state)
    if state:
        imputer = state["imputer_object"]
        if callable(imputer.weights) or callable(imputer.metric):
            return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect the saved neighbor estimator without refitting or normalizing it.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved neighbor estimator without refitting or normalizing it."""
    fields = {"type", "imputer_object", "columns", "n_neighbors", "weights"}
    if local_state_fields(raw, "knn_imputer", fields, allow_empty=True):
        imputer = _validate_local_imputer(raw, KNNImputer)
        _validate_knn_neighbors(imputer, len(raw["columns"]))
    return raw

KNNImputerCalculator

Bases: BaseCalculator

Fit a sklearn KNNImputer on the selected numeric columns.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

@NodeRegistry.register("KNNImputer", KNNImputerApplier)
@node_meta(
    id="KNNImputer",
    name="KNN Imputer",
    category="Preprocessing",
    description="Impute missing values using k-Nearest Neighbors.",
    params={"n_neighbors": 5, "weights": "uniform", "columns": []},
    learns_from_data=True,
)
class KNNImputerCalculator(BaseCalculator):
    """Fit a sklearn ``KNNImputer`` on the selected numeric columns."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with selected columns promoted to ``float64``."""
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> KNNImputerArtifact:  # pylint: disable=arguments-differ
        """Fit the neighbor imputer, dropping all-missing columns sklearn cannot impute."""
        if user_picked_no_columns(config):
            return {}

        n_neighbors = config.get("n_neighbors", 5)
        weights = config.get("weights", "uniform")

        # KNN/Iterative imputers always fit through numpy — engine choice
        # doesn't affect the fit math, so we skip the Pandas hop entirely.
        X_np, cols = resolve_columns_then_to_numpy(X, config, detect_numeric_columns)
        if not cols:
            return {}

        # sklearn silently drops all-missing columns from transform() output,
        # which desyncs the artifact's column list from the imputer's width.
        X_np, cols = drop_all_missing_columns(X_np, cols, "KNNImputerCalculator")
        if not cols:
            return {}

        imputer = KNNImputer(n_neighbors=n_neighbors, weights=weights)
        imputer.fit(X_np)

        return {
            "type": "knn_imputer",
            "imputer_object": imputer,  # Not JSON serializable
            "columns": cols,
            "n_neighbors": n_neighbors,
            "weights": weights,
        }

fit(X, _y, config)

Fit the neighbor imputer, dropping all-missing columns sklearn cannot impute.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> KNNImputerArtifact:  # pylint: disable=arguments-differ
    """Fit the neighbor imputer, dropping all-missing columns sklearn cannot impute."""
    if user_picked_no_columns(config):
        return {}

    n_neighbors = config.get("n_neighbors", 5)
    weights = config.get("weights", "uniform")

    # KNN/Iterative imputers always fit through numpy — engine choice
    # doesn't affect the fit math, so we skip the Pandas hop entirely.
    X_np, cols = resolve_columns_then_to_numpy(X, config, detect_numeric_columns)
    if not cols:
        return {}

    # sklearn silently drops all-missing columns from transform() output,
    # which desyncs the artifact's column list from the imputer's width.
    X_np, cols = drop_all_missing_columns(X_np, cols, "KNNImputerCalculator")
    if not cols:
        return {}

    imputer = KNNImputer(n_neighbors=n_neighbors, weights=weights)
    imputer.fit(X_np)

    return {
        "type": "knn_imputer",
        "imputer_object": imputer,  # Not JSON serializable
        "columns": cols,
        "n_neighbors": n_neighbors,
        "weights": weights,
    }

infer_output_schema(input_schema, config)

Return a schema with selected columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/imputation/knn.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with selected columns promoted to ``float64``."""
    return promote_configured_columns_to_float64(input_schema, config)

LabelEncoderApplier

Bases: BaseApplier

Replace categorical values in place with the integer ids their fitted encoder assigned.

New feature artifacts use typed scalar keys: integral floats match their integer category without matching literal numeric strings. Legacy artifacts retain their original string lookup. Values unseen at fit time map to missing_code. Target encoders retain their existing string-label behavior. Encoded targets are native Series for the feature engine; pandas Series keep their index and all native Series keep their name. Unencoded targets pass through unchanged, including lists and NumPy arrays.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

class LabelEncoderApplier(BaseApplier):
    """Replace categorical values in place with the integer ids their fitted encoder assigned.

    New feature artifacts use typed scalar keys: integral floats match their
    integer category without matching literal numeric strings. Legacy artifacts
    retain their original string lookup. Values unseen at fit time map to
    ``missing_code``. Target encoders retain their existing string-label behavior.
    Encoded targets are native Series for the feature engine; pandas Series keep
    their index and all native Series keep their name. Unencoded targets pass
    through unchanged, including lists and NumPy arrays.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect native learned classes without normalizing the fitted artifact."""
        fields = {"type", "encoders", "columns", "classes_count", "missing_code"}
        if isinstance(raw, dict):
            fields.update(set(raw) & {"category_key_version", "target_column"})
        local_state_fields(raw, "label_encoder", fields)
        uses_category_keys(raw)
        if not isinstance(raw["missing_code"], Decimal):
            local_scalar(raw["missing_code"], "LabelEncoder missing code")
        _validate_label_encoders(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Declare fixed lookups while retaining legacy feature rendering dependencies."""
        if engine not in ("pandas", "polars"):
            return None
        LabelEncoderApplier.validate_inference_state(state)
        legacy = state["columns"] and not uses_category_keys(state)
        context = "global" if engine == "pandas" and legacy else "row"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific encode, forwarding ``(X, y)`` only when ``y`` exists."""
        return apply_target_encoder(
            X,
            y,
            params,
            {"polars": _label_apply_polars, "pandas": _label_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific encode, forwarding (X, y) only when y exists.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific encode, forwarding ``(X, y)`` only when ``y`` exists."""
    return apply_target_encoder(
        X,
        y,
        params,
        {"polars": _label_apply_polars, "pandas": _label_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Declare fixed lookups while retaining legacy feature rendering dependencies.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Declare fixed lookups while retaining legacy feature rendering dependencies."""
    if engine not in ("pandas", "polars"):
        return None
    LabelEncoderApplier.validate_inference_state(state)
    legacy = state["columns"] and not uses_category_keys(state)
    context = "global" if engine == "pandas" and legacy else "row"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect native learned classes without normalizing the fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect native learned classes without normalizing the fitted artifact."""
    fields = {"type", "encoders", "columns", "classes_count", "missing_code"}
    if isinstance(raw, dict):
        fields.update(set(raw) & {"category_key_version", "target_column"})
    local_state_fields(raw, "label_encoder", fields)
    uses_category_keys(raw)
    if not isinstance(raw["missing_code"], Decimal):
        local_scalar(raw["missing_code"], "LabelEncoder missing code")
    _validate_label_encoders(raw)
    return raw

LabelEncoderCalculator

Bases: BaseCalculator

Fit one LabelEncoder per named column, plus one for the target when it qualifies.

Unlike its sibling encoders this node deliberately does not auto-detect feature columns: an empty columns means "encode the target only", mirroring sklearn's convention that LabelEncoder is a 1-D label transformer and OrdinalEncoder is the tool for feature columns. Fitting zero encoders logs a warning rather than passing a silent no-op downstream.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

@NodeRegistry.register("LabelEncoder", LabelEncoderApplier)
@node_meta(
    id="LabelEncoder",
    name="Label Encoder",
    category="Preprocessing",
    description="Encode target labels with value between 0 and n_classes-1.",
    params={"columns": [], "missing_code": -1},
    learns_from_data=True,
)
class LabelEncoderCalculator(BaseCalculator):
    """Fit one LabelEncoder per named column, plus one for the target when it qualifies.

    Unlike its sibling encoders this node deliberately does *not* auto-detect
    feature columns: an empty ``columns`` means "encode the target only",
    mirroring sklearn's convention that ``LabelEncoder`` is a 1-D label
    transformer and ``OrdinalEncoder`` is the tool for feature columns. Fitting
    zero encoders logs a warning rather than passing a silent no-op downstream.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Pass the input schema through: encoded columns keep their name and position."""
        # Label encoding replaces categorical values with ints in place;
        # column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> LabelEncoderArtifact:  # pylint: disable=arguments-differ
        """Fit the encoders on the frame's own engine, forwarding ``y`` when present."""
        return cast(
            LabelEncoderArtifact,
            fit_target_encoder(
                X,
                y,
                config,
                {"polars": _label_fit_polars, "pandas": _label_fit_pandas},
            ),
        )

fit(X, y, config)

Fit the encoders on the frame's own engine, forwarding y when present.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> LabelEncoderArtifact:  # pylint: disable=arguments-differ
    """Fit the encoders on the frame's own engine, forwarding ``y`` when present."""
    return cast(
        LabelEncoderArtifact,
        fit_target_encoder(
            X,
            y,
            config,
            {"polars": _label_fit_polars, "pandas": _label_fit_pandas},
        ),
    )

infer_output_schema(input_schema, config)

Pass the input schema through: encoded columns keep their name and position.

Source code in skyulf-core/skyulf/preprocessing/encoding/label.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Pass the input schema through: encoded columns keep their name and position."""
    # Label encoding replaces categorical values with ints in place;
    # column set is preserved.
    return input_schema

LagFeaturesApplier

Bases: BaseApplier

Append lagged copies of the configured columns, optionally within groups.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

class LagFeaturesApplier(BaseApplier):
    """Append lagged copies of the configured columns, optionally within groups."""

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Retain the window dependency even when training history was saved."""
        if engine not in ("pandas", "polars"):
            return None
        effect = "filter" if state.get("drop_na", False) else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "window")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Add lag columns; ``drop_na`` drops rows containing nulls or floating NaN."""
        if any(
            not isinstance(lag, Integral) or isinstance(lag, bool) or lag <= 0
            for lag in params.get("lags", [])
        ):
            raise ValueError("LagFeatures artifact lags must be positive integers.")
        return apply_dual_engine(
            (X, _y) if _y is not None else X,
            params,
            {"polars": _apply_polars, "pandas": _apply_pandas},
        )

apply(X, _y, params)

Add lag columns; drop_na drops rows containing nulls or floating NaN.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Add lag columns; ``drop_na`` drops rows containing nulls or floating NaN."""
    if any(
        not isinstance(lag, Integral) or isinstance(lag, bool) or lag <= 0
        for lag in params.get("lags", [])
    ):
        raise ValueError("LagFeatures artifact lags must be positive integers.")
    return apply_dual_engine(
        (X, _y) if _y is not None else X,
        params,
        {"polars": _apply_polars, "pandas": _apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Retain the window dependency even when training history was saved.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Retain the window dependency even when training history was saved."""
    if engine not in ("pandas", "polars"):
        return None
    effect = "filter" if state.get("drop_na", False) else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "window")

LagFeaturesCalculator

Bases: BaseCalculator

Save lag configuration and optional bounded training history.

Default batch mode uses only supplied rows. Explicit carry mode seeds future batches from training observations, without mutating the artifact on apply.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

@NodeRegistry.register("LagFeatures", LagFeaturesApplier)
@node_meta(
    id="LagFeatures",
    name="Lag Features",
    category="Preprocessing",
    description="Create lagged copies of columns to expose past values (time series).",
    params={"columns": [], "lags": [1], "group_by": None, "sort_by": None, "drop_na": False},
    tags=["time-series"],
    learns_from_data=False,
)
class LagFeaturesCalculator(BaseCalculator):
    """Save lag configuration and optional bounded training history.

    Default batch mode uses only supplied rows. Explicit carry mode seeds future
    batches from training observations, without mutating the artifact on apply.
    """

    def fit(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        config: dict[str, Any],
    ) -> LagFeaturesArtifact:
        """Record the columns, deduplicated positive lags, and sort/group/drop settings."""
        params: dict[str, Any] = {
            "type": "lag_features",
            "columns": config.get("columns", []),
            "lags": coerce_lags(config.get("lags", [1])),
            "group_by": config.get("group_by"),
            "sort_by": config.get("sort_by"),
            "drop_na": bool(config.get("drop_na", False)),
        }
        return cast(
            LagFeaturesArtifact, fit_history(df, config, params, max(params["lags"], default=0))
        )

    def fit_transform_train(self, df: Any, config: dict[str, Any]) -> tuple[Any, Any]:
        """Save the training tail while computing training lags without future context."""
        params = self.fit(df, config)
        return params, LagFeaturesApplier().apply(df, dict(params) | {"_history_training": True})

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Add one ``{col}_lag_{n}`` column per lag, as ``float64``."""
        # Shifted lag columns are nullable by construction; pandas/polars keep
        # them as ``float64`` because of the inserted missing row.
        cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
        lags = coerce_lags(config.get("lags", [1]))
        schema = input_schema
        for col in cols:
            for lag in lags:
                schema = schema.add(_lag_name(col, lag), "float64")
        return schema

fit(df, config)

Record the columns, deduplicated positive lags, and sort/group/drop settings.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

def fit(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    config: dict[str, Any],
) -> LagFeaturesArtifact:
    """Record the columns, deduplicated positive lags, and sort/group/drop settings."""
    params: dict[str, Any] = {
        "type": "lag_features",
        "columns": config.get("columns", []),
        "lags": coerce_lags(config.get("lags", [1])),
        "group_by": config.get("group_by"),
        "sort_by": config.get("sort_by"),
        "drop_na": bool(config.get("drop_na", False)),
    }
    return cast(
        LagFeaturesArtifact, fit_history(df, config, params, max(params["lags"], default=0))
    )

fit_transform_train(df, config)

Save the training tail while computing training lags without future context.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

def fit_transform_train(self, df: Any, config: dict[str, Any]) -> tuple[Any, Any]:
    """Save the training tail while computing training lags without future context."""
    params = self.fit(df, config)
    return params, LagFeaturesApplier().apply(df, dict(params) | {"_history_training": True})

infer_output_schema(input_schema, config)

Add one {col}_lag_{n} column per lag, as float64.

Source code in skyulf-core/skyulf/preprocessing/time_series/lag.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Add one ``{col}_lag_{n}`` column per lag, as ``float64``."""
    # Shifted lag columns are nullable by construction; pandas/polars keep
    # them as ``float64`` because of the inserted missing row.
    cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
    lags = coerce_lags(config.get("lags", [1]))
    schema = input_schema
    for col in cols:
        for lag in lags:
            schema = schema.add(_lag_name(col, lag), "float64")
    return schema

MergedBranchFoldAdapter

Re-runs fork-join preprocessing branches inside each CV/tuning fold.

Built for graphs where a shared trunk ends in a splitter (fork point) and N parallel transformer branches fan back into one training node. Per fold it re-runs every branch step list on the fold-train payload and merges the branch frames exactly like the engine's pure-strategy column-wise merge, so the fold sees the same columns the full run produces — without any pre-fit statistics leaking from outside the fold.

Parameters:

Name	Type	Description	Default
branch_step_lists	list[list[dict[str, Any]]]	One unfitted step list per branch, in the engine's merge input order.	required
merge_strategy	str	"last_wins" (default engine behaviour) or "first_wins" — must match the training node's configured strategy so fold columns match the full-run merge.	required
target_column	str	Target name, kept to reject payloads that still embed it; target-aware branch steps receive y through the payload.	required
drop_columns	list[str] | tuple[str, ...]	Columns removed upstream (e.g. by Drop Columns nodes); stripped again after each merge so a branch cannot resurrect them.	()

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

class MergedBranchFoldAdapter:
    """Re-runs fork-join preprocessing branches inside each CV/tuning fold.

    Built for graphs where a shared trunk ends in a splitter (fork point) and
    N parallel transformer branches fan back into one training node. Per fold
    it re-runs every branch step list on the fold-train payload and merges the
    branch frames exactly like the engine's pure-strategy column-wise merge,
    so the fold sees the same columns the full run produces — without any
    pre-fit statistics leaking from outside the fold.

    Args:
        branch_step_lists: One unfitted step list per branch, in the engine's
            merge input order.
        merge_strategy: ``"last_wins"`` (default engine behaviour) or
            ``"first_wins"`` — must match the training node's configured
            strategy so fold columns match the full-run merge.
        target_column: Target name, kept to reject payloads that still embed
            it; target-aware branch steps receive ``y`` through the payload.
        drop_columns: Columns removed upstream (e.g. by Drop Columns nodes);
            stripped again after each merge so a branch cannot resurrect them.
    """

    def __init__(
        self,
        branch_step_lists: list[list[dict[str, Any]]],
        merge_strategy: str,
        target_column: str,
        drop_columns: list[str] | tuple[str, ...] = (),
    ):
        """Validate every branch against the positional merge's row contract.

        Each step list is trial-built as a :class:`FeatureEngineer`, screened
        for splitting, row filtering and configured sorting, and resolved
        through the node registry. Configurations are copied in full so later
        caller edits cannot bypass these checks.

        Raises:
            ValueError: If the merge strategy is unrecognized, if there is no
                branch step list or one is empty, or if a branch step splits the
                data, changes row counts or sorts rows.
        """
        if merge_strategy not in ("last_wins", "first_wins"):
            raise ValueError(f"unknown merge strategy '{merge_strategy}'")
        if not branch_step_lists:
            raise ValueError("at least one branch step list is required")
        for steps in branch_step_lists:
            if not steps:
                raise ValueError("branch step list must not be empty")
            FeatureEngineer(list(steps))
            for step in steps:
                transformer = step["transformer"]
                reason = merged_branch_step_unsafe_reason(step)
                if reason is not None:
                    raise ValueError(
                        f"branch step '{transformer}' cannot run inside a merged fold: it {reason}"
                    )
                NodeRegistry.get_calculator(transformer)
        self._branch_step_lists = deepcopy(branch_step_lists)
        self._merge_strategy = merge_strategy
        self._target_column = target_column
        self._drop_columns = list(drop_columns)
        self._engineers: list[FeatureEngineer] | None = None
        # Branch configurations are screened for filtering and sorting, so
        # the positional merge keeps every observation in its original order.
        self.changes_row_count = False

    def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
        """Fit fresh branch engineers on this fold's payload and merge the results.

        Every branch engineer is rebuilt from the stored step lists on each call
        rather than reused, so a statistic fitted on one fold cannot reach the
        next.

        Raises:
            ValueError: If ``X`` still embeds the configured target column.
        """
        self.train_sample_weight_ = prepare_weights(sample_weight, frame_rows(X), self)
        self._validate_payload(X)
        engineers = [FeatureEngineer(list(steps)) for steps in self._branch_step_lists]
        frames, ys = self._run_branches(engineers, (X, y), fit=True)
        result = self._finalize(frames, ys)
        self._engineers = engineers
        return result

    def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
        """Re-apply the branch engineers fitted by the last :meth:`fit_transform`.

        Raises:
            RuntimeError: If called before :meth:`fit_transform`.
            ValueError: If ``X`` still embeds the configured target column.
        """
        if self._engineers is None:
            raise RuntimeError("transform() called before fit_transform()")
        self._validate_payload(X)
        payload = (X, y) if y is not None else X
        frames, ys = self._run_branches(self._engineers, payload, fit=False)
        return self._finalize(frames, ys)

    def _run_branches(
        self, engineers: list[FeatureEngineer], payload: Any, *, fit: bool
    ) -> tuple[list[pd.DataFrame], list[Any]]:
        """Run each branch and reject row-count violations before a positional merge."""
        frames: list[pd.DataFrame] = []
        ys: list[Any] = []
        input_y = payload[1] if isinstance(payload, tuple) else None
        input_X = payload[0] if isinstance(payload, tuple) else payload
        expected_rows = frame_rows(input_X)
        if input_y is not None and frame_rows(input_y) != expected_rows:
            raise ValueError("Merged fold features and target have different row counts")
        for engineer in engineers:
            out = (
                engineer.fit_transform(payload, target_column=self._target_column)[0]
                if fit
                else engineer.transform(payload)
            )
            frame, y_out = self._validated_branch_output(out, input_y, expected_rows)
            frames.append(frame)
            ys.append(y_out)
        return frames, ys

    @staticmethod
    def _validated_branch_output(
        out: Any, input_y: Any, expected_rows: int
    ) -> tuple[pd.DataFrame, Any]:
        """Unpack a branch and validate feature and target lengths before conversion."""
        if isinstance(out, tuple) and len(out) == 2:
            frame, y_out = out
        else:
            # Appliers that return a bare frame pass the target through.
            frame, y_out = out, input_y
        if frame_rows(frame) != expected_rows or (
            y_out is not None and frame_rows(y_out) != expected_rows
        ):
            raise ValueError("Merged fold branch changed row counts; positional merge is unsafe")
        if isinstance(frame, pl.DataFrame):
            frame = frame.to_pandas()
        return frame, y_out

    def _finalize(self, frames: list[pd.DataFrame], ys: list[Any]) -> tuple[Any, Any]:
        """Apply column ownership and configured drops to validated branch outputs."""
        merged = _merge_branch_frames_columnwise(frames, self._merge_strategy)
        drop = [col for col in self._drop_columns if col in merged.columns]
        if drop:
            merged = merged.drop(columns=drop)
        # Mirrors the engine's SplitDataset merge: the first branch supplies
        # the target (branches are screened to never change row counts).
        return merged, ys[0]

    def _validate_payload(self, X: Any) -> None:
        """Reject targets embedded in the feature columns before branch execution."""
        if hasattr(X, "columns") and self._target_column in X.columns:
            raise ValueError(f"target column '{self._target_column}' already present in X")

    def original_target_labels(self, labels: Any) -> Any:
        """Decode the first branch, which supplies the merged fold's target."""
        if self._engineers is None:
            raise RuntimeError("Target decoding called before fit_transform()")
        return original_target_labels(self._engineers[0].fitted_steps, labels)

__init__(branch_step_lists, merge_strategy, target_column, drop_columns=())

Validate every branch against the positional merge's row contract.

Each step list is trial-built as a :class:FeatureEngineer, screened for splitting, row filtering and configured sorting, and resolved through the node registry. Configurations are copied in full so later caller edits cannot bypass these checks.

Raises:

Type	Description
ValueError	If the merge strategy is unrecognized, if there is no branch step list or one is empty, or if a branch step splits the data, changes row counts or sorts rows.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def __init__(
    self,
    branch_step_lists: list[list[dict[str, Any]]],
    merge_strategy: str,
    target_column: str,
    drop_columns: list[str] | tuple[str, ...] = (),
):
    """Validate every branch against the positional merge's row contract.

    Each step list is trial-built as a :class:`FeatureEngineer`, screened
    for splitting, row filtering and configured sorting, and resolved
    through the node registry. Configurations are copied in full so later
    caller edits cannot bypass these checks.

    Raises:
        ValueError: If the merge strategy is unrecognized, if there is no
            branch step list or one is empty, or if a branch step splits the
            data, changes row counts or sorts rows.
    """
    if merge_strategy not in ("last_wins", "first_wins"):
        raise ValueError(f"unknown merge strategy '{merge_strategy}'")
    if not branch_step_lists:
        raise ValueError("at least one branch step list is required")
    for steps in branch_step_lists:
        if not steps:
            raise ValueError("branch step list must not be empty")
        FeatureEngineer(list(steps))
        for step in steps:
            transformer = step["transformer"]
            reason = merged_branch_step_unsafe_reason(step)
            if reason is not None:
                raise ValueError(
                    f"branch step '{transformer}' cannot run inside a merged fold: it {reason}"
                )
            NodeRegistry.get_calculator(transformer)
    self._branch_step_lists = deepcopy(branch_step_lists)
    self._merge_strategy = merge_strategy
    self._target_column = target_column
    self._drop_columns = list(drop_columns)
    self._engineers: list[FeatureEngineer] | None = None
    # Branch configurations are screened for filtering and sorting, so
    # the positional merge keeps every observation in its original order.
    self.changes_row_count = False

fit_transform(X, y, *, sample_weight=None)

Fit fresh branch engineers on this fold's payload and merge the results.

Every branch engineer is rebuilt from the stored step lists on each call rather than reused, so a statistic fitted on one fold cannot reach the next.

Raises:

Type	Description
ValueError	If X still embeds the configured target column.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def fit_transform(self, X: Any, y: Any, *, sample_weight: Any = None) -> tuple[Any, Any]:
    """Fit fresh branch engineers on this fold's payload and merge the results.

    Every branch engineer is rebuilt from the stored step lists on each call
    rather than reused, so a statistic fitted on one fold cannot reach the
    next.

    Raises:
        ValueError: If ``X`` still embeds the configured target column.
    """
    self.train_sample_weight_ = prepare_weights(sample_weight, frame_rows(X), self)
    self._validate_payload(X)
    engineers = [FeatureEngineer(list(steps)) for steps in self._branch_step_lists]
    frames, ys = self._run_branches(engineers, (X, y), fit=True)
    result = self._finalize(frames, ys)
    self._engineers = engineers
    return result

original_target_labels(labels)

Decode the first branch, which supplies the merged fold's target.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def original_target_labels(self, labels: Any) -> Any:
    """Decode the first branch, which supplies the merged fold's target."""
    if self._engineers is None:
        raise RuntimeError("Target decoding called before fit_transform()")
    return original_target_labels(self._engineers[0].fitted_steps, labels)

transform(X, y)

Re-apply the branch engineers fitted by the last :meth:fit_transform.

Raises:

Type	Description
RuntimeError	If called before :meth:fit_transform.
ValueError	If X still embeds the configured target column.

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def transform(self, X: Any, y: Any) -> tuple[Any, Any]:
    """Re-apply the branch engineers fitted by the last :meth:`fit_transform`.

    Raises:
        RuntimeError: If called before :meth:`fit_transform`.
        ValueError: If ``X`` still embeds the configured target column.
    """
    if self._engineers is None:
        raise RuntimeError("transform() called before fit_transform()")
    self._validate_payload(X)
    payload = (X, y) if y is not None else X
    frames, ys = self._run_branches(self._engineers, payload, fit=False)
    return self._finalize(frames, ys)

MinMaxScalerApplier

Bases: BaseApplier

Rescale selected columns or replay an explicit empty selection unchanged.

Empty artifacts declare identity only with an explicit empty column recipe. Automatic selections and non-finite learned coefficients remain unreviewed.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

class MinMaxScalerApplier(BaseApplier):
    """Rescale selected columns or replay an explicit empty selection unchanged.

    Empty artifacts declare identity only with an explicit empty column recipe.
    Automatic selections and non-finite learned coefficients remain unreviewed.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return _minmax_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        if not state and (not user_picked_no_columns(raw) or raw.get("_auto_columns")):
            raise ValueError("An empty scaler artifact requires explicit empty columns.")
        return _minmax_config(fitted_columns(_minmax_values(raw), state or {"columns": []}), state)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Apply the fitted affine scale/shift to ``X``; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        min_val = params.get("min")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        if not valid or min_val is None or scale is None:
            return X, _y

        exprs = [
            (pl.col(c) * scale[cols.index(c)] + min_val[cols.index(c)]).alias(c) for c in valid
        ]
        return X.with_columns(exprs), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        min_val = params.get("min")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        if not valid or min_val is None or scale is None:
            return X, _y

        X_out = X.copy()
        col_indices = [cols.index(c) for c in valid]
        subset = decimal_columns_to_float(X_out[valid], valid)
        vals = subset.to_numpy(dtype=np.float64, na_value=np.nan)
        vals = vals * np.array(scale)[col_indices] + np.array(min_val)[col_indices]
        X_out[valid] = vals
        return X_out, _y

apply(X, _y, params)

Apply the fitted affine scale/shift to X; y passes through.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Apply the fitted affine scale/shift to ``X``; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    if not state and (not user_picked_no_columns(raw) or raw.get("_auto_columns")):
        raise ValueError("An empty scaler artifact requires explicit empty columns.")
    return _minmax_config(fitted_columns(_minmax_values(raw), state or {"columns": []}), state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return _minmax_state(raw)

MinMaxScalerCalculator

Bases: BaseCalculator

Fit scale/offset statistics that map the data into feature_range.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

@NodeRegistry.register(
    "MinMaxScaler",
    MinMaxScalerApplier,
    execution_capabilities=(
        ExecutionCapability("pandas", "apply", "python_batch", "preserve", "row"),
        ExecutionCapability("polars", "apply", "local", "preserve", "row"),
    ),
)
@node_meta(
    id="MinMaxScaler",
    name="Min-Max Scaler",
    category="Preprocessing",
    description="Transform features by scaling each feature to a given range.",
    params={"feature_range": [0, 1], "columns": []},
    learns_from_data=True,
)
class MinMaxScalerCalculator(BaseCalculator):
    """Fit scale/offset statistics that map the data into ``feature_range``."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with transformed columns promoted to ``float64``."""
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> MinMaxScalerArtifact:  # pylint: disable=arguments-differ
        """Dispatch the fit to the active engine and return the scaler-statistics artifact."""
        if user_picked_no_columns(config):
            return cast(MinMaxScalerArtifact, {})
        return cast(
            MinMaxScalerArtifact,
            fit_dual_engine(X, config, {"polars": self._fit_polars, "pandas": self._fit_pandas}),
        )

    @staticmethod
    def _fit_polars(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_polars(X, config)
        if not cols:
            return {}
        return _fit_minmax(X_subset, cols, config)

    @staticmethod
    def _fit_pandas(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_pandas(X, config)
        if not cols:
            return {}
        return _fit_minmax(X_subset, cols, config)

fit(X, _y, config)

Dispatch the fit to the active engine and return the scaler-statistics artifact.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> MinMaxScalerArtifact:  # pylint: disable=arguments-differ
    """Dispatch the fit to the active engine and return the scaler-statistics artifact."""
    if user_picked_no_columns(config):
        return cast(MinMaxScalerArtifact, {})
    return cast(
        MinMaxScalerArtifact,
        fit_dual_engine(X, config, {"polars": self._fit_polars, "pandas": self._fit_pandas}),
    )

infer_output_schema(input_schema, config)

Return a schema with transformed columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/scaling/minmax.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with transformed columns promoted to ``float64``."""
    return promote_configured_columns_to_float64(input_schema, config)

MissingIndicatorApplier

Bases: BaseApplier

Add a binary <col><flag_suffix> column beside each configured column.

Additive, not replacing: the original columns survive. Both engines must flag the same cells, which needs care on polars — it stores float NaN distinctly from null where pandas' isna() treats both as missing, so float columns get an explicit NaN test. Columns absent from the frame are skipped. A generated flag name matching an existing column raises ValueError; input values are never overwritten by flags.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

class MissingIndicatorApplier(BaseApplier):
    """Add a binary ``<col><flag_suffix>`` column beside each configured column.

    Additive, not replacing: the original columns survive. Both engines must
    flag the same cells, which needs care on polars — it stores float NaN
    distinctly from null where pandas' ``isna()`` treats both as missing, so
    float columns get an explicit NaN test. Columns absent from the frame are
    skipped. A generated flag name matching an existing column raises
    ``ValueError``; input values are never overwritten by flags.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned flag selection without detecting missing values again."""
        local_state_fields(raw, "missing_indicator", {"type", "columns", "flag_suffix"})
        columns = _columns(raw["columns"])
        if type(raw["flag_suffix"]) is not str or not raw["flag_suffix"]:
            raise ValueError("Fitted missing-indicator suffix must be a nonempty string.")
        validate_generated_column_names(
            columns,
            (f"{column}{raw['flag_suffix']}" for column in columns),
            node_name="MissingIndicator",
        )
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local saved apply context without granting worker execution."""
        if engine not in ("pandas", "polars"):
            return None
        MissingIndicatorApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch the flag creation to the pandas or polars path; ``y`` passes through."""
        return apply_dual_engine(
            X,
            params,
            {"polars": _missing_indicator_apply_polars, "pandas": _missing_indicator_apply_pandas},
        )

apply(X, _y, params)

Dispatch the flag creation to the pandas or polars path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch the flag creation to the pandas or polars path; ``y`` passes through."""
    return apply_dual_engine(
        X,
        params,
        {"polars": _missing_indicator_apply_polars, "pandas": _missing_indicator_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe local saved apply context without granting worker execution.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local saved apply context without granting worker execution."""
    if engine not in ("pandas", "polars"):
        return None
    MissingIndicatorApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect learned flag selection without detecting missing values again.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned flag selection without detecting missing values again."""
    local_state_fields(raw, "missing_indicator", {"type", "columns", "flag_suffix"})
    columns = _columns(raw["columns"])
    if type(raw["flag_suffix"]) is not str or not raw["flag_suffix"]:
        raise ValueError("Fitted missing-indicator suffix must be a nonempty string.")
    validate_generated_column_names(
        columns,
        (f"{column}{raw['flag_suffix']}" for column in columns),
        node_name="MissingIndicator",
    )
    return raw

MissingIndicatorCalculator

Bases: BaseCalculator

Choose which columns get a missing flag, learning the set from data when none is named.

An explicit columns list is filtered down to names present on the frame; without one, every column that actually contains a missing value is flagged, so the artifact depends on the training data.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

@NodeRegistry.register("MissingIndicator", MissingIndicatorApplier)
@node_meta(
    id="MissingIndicator",
    name="Missing Indicator",
    category="Feature Engineering",
    description="Create binary indicators for missing values.",
    params={"columns": [], "flag_suffix": _DEFAULT_FLAG_SUFFIX},
    learns_from_data=True,
)
class MissingIndicatorCalculator(BaseCalculator):
    """Choose which columns get a missing flag, learning the set from data when none is named.

    An explicit ``columns`` list is filtered down to names present on the
    frame; without one, every column that actually contains a missing value is
    flagged, so the artifact depends on the training data.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Predict the added flag columns, or ``None`` when the set is data-dependent.

        With an explicit list the output is the input plus one ``int64``
        ``<col><flag_suffix>`` column for each source present in the input;
        otherwise which columns hold missing values decides, and callers
        must introspect at runtime.
        """
        # Adds one int64 (0/1) column "<col><flag_suffix>" per indicator
        # column. Only predictable when the user supplied an explicit column
        # list; otherwise the set depends on which columns actually contain
        # missing values.
        explicit = config.get("columns") or []
        if not explicit:
            return None
        suffix = config.get("flag_suffix") or _DEFAULT_FLAG_SUFFIX
        new_schema = input_schema
        for col in explicit:
            if col in input_schema:
                new_schema = new_schema.add(f"{col}{suffix}", "int64")
        return new_schema

    def fit(self, df: Any, config: dict[str, Any]) -> MissingIndicatorArtifact:
        """Compute the flagged column set on the frame's own engine and return it."""
        return cast(
            MissingIndicatorArtifact,
            fit_dual_engine(
                df,
                config,
                {"polars": _missing_indicator_fit_polars, "pandas": _missing_indicator_fit_pandas},
            ),
        )

fit(df, config)

Compute the flagged column set on the frame's own engine and return it.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

def fit(self, df: Any, config: dict[str, Any]) -> MissingIndicatorArtifact:
    """Compute the flagged column set on the frame's own engine and return it."""
    return cast(
        MissingIndicatorArtifact,
        fit_dual_engine(
            df,
            config,
            {"polars": _missing_indicator_fit_polars, "pandas": _missing_indicator_fit_pandas},
        ),
    )

infer_output_schema(input_schema, config)

Predict the added flag columns, or None when the set is data-dependent.

With an explicit list the output is the input plus one int64 <col><flag_suffix> column for each source present in the input; otherwise which columns hold missing values decides, and callers must introspect at runtime.

Source code in skyulf-core/skyulf/preprocessing/drop_and_missing/missing_indicator.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Predict the added flag columns, or ``None`` when the set is data-dependent.

    With an explicit list the output is the input plus one ``int64``
    ``<col><flag_suffix>`` column for each source present in the input;
    otherwise which columns hold missing values decides, and callers
    must introspect at runtime.
    """
    # Adds one int64 (0/1) column "<col><flag_suffix>" per indicator
    # column. Only predictable when the user supplied an explicit column
    # list; otherwise the set depends on which columns actually contain
    # missing values.
    explicit = config.get("columns") or []
    if not explicit:
        return None
    suffix = config.get("flag_suffix") or _DEFAULT_FLAG_SUFFIX
    new_schema = input_schema
    for col in explicit:
        if col in input_schema:
            new_schema = new_schema.add(f"{col}{suffix}", "int64")
    return new_schema

ModelBasedSelectionApplier

Bases: BaseApplier

Remove the columns an estimator's importances rejected.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

class ModelBasedSelectionApplier(BaseApplier):
    """Remove the columns an estimator's importances rejected."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved membership while leaving unused fit reports unchanged."""
        fields = {
            "type",
            "selected_columns",
            "candidate_columns",
            "method",
            "drop_columns",
            "feature_importances",
        }
        if local_state_fields(raw, "model_based_selection", fields, allow_empty=True):
            selected = _local_selection_columns(raw["selected_columns"])
            candidates = _local_selection_columns(raw["candidate_columns"])
            if not set(selected).issubset(candidates):
                raise ValueError("Fitted selected columns must belong to candidate columns.")
            _local_selection_drop_flag(raw["drop_columns"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved column selection without rerunning the training estimator."""
        if engine not in {"pandas", "polars"}:
            return None
        ModelBasedSelectionApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Drop ``candidate_columns - selected_columns`` from ``X``.

        Honours the artifact's ``drop_columns`` flag; the recorded
        ``feature_importances`` are reporting metadata and take no part in the
        transform.
        """
        return apply_dual_engine(
            X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
        )

apply(X, _y, params)

Drop candidate_columns - selected_columns from X.

Honours the artifact's drop_columns flag; the recorded feature_importances are reporting metadata and take no part in the transform.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Drop ``candidate_columns - selected_columns`` from ``X``.

    Honours the artifact's ``drop_columns`` flag; the recorded
    ``feature_importances`` are reporting metadata and take no part in the
    transform.
    """
    return apply_dual_engine(
        X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved column selection without rerunning the training estimator.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved column selection without rerunning the training estimator."""
    if engine not in {"pandas", "polars"}:
        return None
    ModelBasedSelectionApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved membership while leaving unused fit reports unchanged.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved membership while leaving unused fit reports unchanged."""
    fields = {
        "type",
        "selected_columns",
        "candidate_columns",
        "method",
        "drop_columns",
        "feature_importances",
    }
    if local_state_fields(raw, "model_based_selection", fields, allow_empty=True):
        selected = _local_selection_columns(raw["selected_columns"])
        candidates = _local_selection_columns(raw["candidate_columns"])
        if not set(selected).issubset(candidates):
            raise ValueError("Fitted selected columns must belong to candidate columns.")
        _local_selection_drop_flag(raw["drop_columns"])
    return raw

ModelBasedSelectionCalculator

Bases: BaseCalculator

Fit a SelectFromModel or RFE selector around a resolved estimator.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

@NodeRegistry.register("ModelBasedSelection", ModelBasedSelectionApplier)
@node_meta(
    id="ModelBasedSelection",
    name="Model-Based Selection",
    category="Feature Selection",
    description="Select features based on importance weights.",
    params={"estimator": "RandomForest", "threshold": "mean", "max_features": None},
    learns_from_data=True,
)
class ModelBasedSelectionCalculator(BaseCalculator):
    """Fit a ``SelectFromModel`` or ``RFE`` selector around a resolved estimator."""

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> ModelBasedSelectionArtifact:  # pylint: disable=arguments-differ
        """Return the features the fitted estimator's support keeps.

        This selector is supervised: with no target there is nothing to score
        against, so a missing target is treated as an error. Every failure mode
        returns an empty artifact instead of raising, which degrades the node to
        a passthrough — though an unsupported ``method`` does so silently,
        without the log line the other branches emit.
        """
        target_col = config.get("target_column")
        # Resolve candidate columns natively on the raw frame first (works on
        # Polars without conversion), then convert only the columns actually
        # needed -- candidates plus the target column when it must be pulled
        # out of X itself (y is None) -- instead of the whole input frame.
        cols = _resolve_candidate_columns(X, config, target_col)
        if not cols:
            return cast(ModelBasedSelectionArtifact, {})

        needed_cols = cols if y is not None or not target_col else [*cols, target_col]
        X_pd = select_then_to_pandas(X, needed_cols)

        y = _extract_target(X_pd, y, target_col)
        if y is None:
            logger.error(
                f"ModelBasedSelection requires target column '{target_col}' "
                "to be present in training data."
            )
            return cast(ModelBasedSelectionArtifact, {})

        method = config.get("method", "select_from_model")
        estimator_name = config.get("estimator", "auto")
        problem_type = _resolve_problem_type(config.get("problem_type", "auto"), y)

        estimator = _resolve_estimator(estimator_name, problem_type)
        if estimator is None:
            logger.error(
                f"Could not resolve estimator '{estimator_name}' for problem type '{problem_type}'"
            )
            return cast(ModelBasedSelectionArtifact, {})

        selector = _build_model_selector(method, estimator, config)
        if selector is None:
            return cast(ModelBasedSelectionArtifact, {})

        X_np, _ = SklearnBridge.to_sklearn(_fillna_zero_with_warning(X_pd, cols))
        selector.fit(X_np, _prepare_sklearn_y(y, problem_type))
        support = selector.get_support()
        selected_cols = [c for c, s in zip(cols, support, strict=True) if s]

        return cast(
            ModelBasedSelectionArtifact,
            {
                "type": "model_based_selection",
                "selected_columns": selected_cols,
                "candidate_columns": cols,
                "method": method,
                "drop_columns": config.get("drop_columns", True),
                "feature_importances": _model_feature_importances(selector, cols),
            },
        )

fit(X, y, config)

Return the features the fitted estimator's support keeps.

This selector is supervised: with no target there is nothing to score against, so a missing target is treated as an error. Every failure mode returns an empty artifact instead of raising, which degrades the node to a passthrough — though an unsupported method does so silently, without the log line the other branches emit.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/model_based.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> ModelBasedSelectionArtifact:  # pylint: disable=arguments-differ
    """Return the features the fitted estimator's support keeps.

    This selector is supervised: with no target there is nothing to score
    against, so a missing target is treated as an error. Every failure mode
    returns an empty artifact instead of raising, which degrades the node to
    a passthrough — though an unsupported ``method`` does so silently,
    without the log line the other branches emit.
    """
    target_col = config.get("target_column")
    # Resolve candidate columns natively on the raw frame first (works on
    # Polars without conversion), then convert only the columns actually
    # needed -- candidates plus the target column when it must be pulled
    # out of X itself (y is None) -- instead of the whole input frame.
    cols = _resolve_candidate_columns(X, config, target_col)
    if not cols:
        return cast(ModelBasedSelectionArtifact, {})

    needed_cols = cols if y is not None or not target_col else [*cols, target_col]
    X_pd = select_then_to_pandas(X, needed_cols)

    y = _extract_target(X_pd, y, target_col)
    if y is None:
        logger.error(
            f"ModelBasedSelection requires target column '{target_col}' "
            "to be present in training data."
        )
        return cast(ModelBasedSelectionArtifact, {})

    method = config.get("method", "select_from_model")
    estimator_name = config.get("estimator", "auto")
    problem_type = _resolve_problem_type(config.get("problem_type", "auto"), y)

    estimator = _resolve_estimator(estimator_name, problem_type)
    if estimator is None:
        logger.error(
            f"Could not resolve estimator '{estimator_name}' for problem type '{problem_type}'"
        )
        return cast(ModelBasedSelectionArtifact, {})

    selector = _build_model_selector(method, estimator, config)
    if selector is None:
        return cast(ModelBasedSelectionArtifact, {})

    X_np, _ = SklearnBridge.to_sklearn(_fillna_zero_with_warning(X_pd, cols))
    selector.fit(X_np, _prepare_sklearn_y(y, problem_type))
    support = selector.get_support()
    selected_cols = [c for c, s in zip(cols, support, strict=True) if s]

    return cast(
        ModelBasedSelectionArtifact,
        {
            "type": "model_based_selection",
            "selected_columns": selected_cols,
            "candidate_columns": cols,
            "method": method,
            "drop_columns": config.get("drop_columns", True),
            "feature_importances": _model_feature_importances(selector, cols),
        },
    )

OneHotEncoderApplier

Bases: BaseApplier

Expand each categorical column into its one-hot indicator columns.

Parity here comes from both engines transforming through the same fitted sklearn encoder carried in the artifact, not from two parallel implementations. Nulls are filled with the sentinel token before transforming, which only lines up when include_missing was set at fit time too — the encoder knows that level solely if it saw it. Indicator columns are concatenated on and the originals dropped unless drop_original is false. Generated names must be unique and must not collide with retained input columns; conflicts raise ValueError. New artifacts escape literal missing tokens and escape-prefixed strings; unversioned artifacts retain their original missing-value transformation. An empty artifact declares identity replay only for an explicit empty column selection with a validated recipe.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

class OneHotEncoderApplier(BaseApplier):
    """Expand each categorical column into its one-hot indicator columns.

    Parity here comes from both engines transforming through the *same* fitted
    sklearn encoder carried in the artifact, not from two parallel
    implementations. Nulls are filled with the sentinel token before
    transforming, which only lines up when ``include_missing`` was set at fit
    time too — the encoder knows that level solely if it saw it. Indicator
    columns are concatenated on and the originals dropped unless
    ``drop_original`` is false. Generated names must be unique and must not
    collide with retained input columns; conflicts raise ``ValueError``.
    New artifacts escape literal missing tokens and escape-prefixed strings;
    unversioned artifacts retain their original missing-value transformation.
    An empty artifact declares identity replay only for an explicit empty
    column selection with a validated recipe.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return _onehot_state(raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        if not state and (not user_picked_no_columns(raw) or raw.get("_auto_columns")):
            raise ValueError("An empty encoder artifact requires explicit empty columns.")
        return _onehot_config(fitted_columns(raw, state or {"columns": []}), state)

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific transform, forwarding ``(X, y)`` only when present."""
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"polars": _onehot_apply_polars, "pandas": _onehot_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific transform, forwarding (X, y) only when present.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific transform, forwarding ``(X, y)`` only when present."""
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"polars": _onehot_apply_polars, "pandas": _onehot_apply_pandas},
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    if not state and (not user_picked_no_columns(raw) or raw.get("_auto_columns")):
        raise ValueError("An empty encoder artifact requires explicit empty columns.")
    return _onehot_config(fitted_columns(raw, state or {"columns": []}), state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return _onehot_state(raw)

OneHotEncoderCalculator

Bases: BaseCalculator

Fit a sklearn OneHotEncoder on the resolved categorical columns.

The target column is excluded before fitting: one-hot replaces the column it encodes with several derived ones, which would break a downstream Feature/Target Split. An explicit columns: [], or auto-detection finding no categorical columns, yields an empty artifact so the applier no-ops. Degenerate columns (zero categories, or one category with drop_first) are warned about rather than silently producing nothing.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

@NodeRegistry.register(
    "OneHotEncoder",
    OneHotEncoderApplier,
    execution_capabilities=(
        ExecutionCapability(
            "pandas",
            "apply",
            "python_batch",
            "preserve",
            "row",
            config_match=(("max_categories", None), ("include_missing", False)),
        ),
        ExecutionCapability("pandas", "apply", "local", "preserve", "row"),
        ExecutionCapability("polars", "apply", "local", "preserve", "row"),
    ),
)
@node_meta(
    id="OneHotEncoder",
    name="One-Hot Encoder",
    category="Preprocessing",
    description="Encodes categorical features as a one-hot numeric array.",
    params={
        "handle_unknown": "ignore",
        "drop_first": False,
        "max_categories": 20,
        "columns": [],
        "include_missing": False,
    },
    learns_from_data=True,
)
class OneHotEncoderCalculator(BaseCalculator):
    """Fit a sklearn ``OneHotEncoder`` on the resolved categorical columns.

    The target column is excluded before fitting: one-hot replaces the column
    it encodes with several derived ones, which would break a downstream
    Feature/Target Split. An explicit ``columns: []``, or auto-detection
    finding no categorical columns, yields an empty artifact so the applier
    no-ops. Degenerate columns (zero categories, or one category with
    ``drop_first``) are warned about rather than silently producing nothing.
    """

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> OneHotArtifact:  # pylint: disable=arguments-differ
        """Short-circuit an explicit empty column selection, else fit on the frame's own engine."""
        if user_picked_no_columns(config):
            return {}
        return cast(
            OneHotArtifact,
            fit_dual_engine(
                (X, y) if y is not None else X,
                config,
                {"polars": _onehot_fit_polars, "pandas": _onehot_fit_pandas},
            ),
        )

fit(X, y, config)

Short-circuit an explicit empty column selection, else fit on the frame's own engine.

Source code in skyulf-core/skyulf/preprocessing/encoding/one_hot.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> OneHotArtifact:  # pylint: disable=arguments-differ
    """Short-circuit an explicit empty column selection, else fit on the frame's own engine."""
    if user_picked_no_columns(config):
        return {}
    return cast(
        OneHotArtifact,
        fit_dual_engine(
            (X, y) if y is not None else X,
            config,
            {"polars": _onehot_fit_polars, "pandas": _onehot_fit_pandas},
        ),
    )

OrdinalEncoderApplier

Bases: BaseApplier

Replace categorical values in place with fitted ordinal indices, target included.

Features use the shared encoder_object and versioned scalar keys; y uses a separate __target__ encoder with its existing string-label rules. Older feature artifacts retain their original string lookup. An unseen value becomes unknown_value unless handle_unknown is "error". Encoded output is float32, not integer.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

class OrdinalEncoderApplier(BaseApplier):
    """Replace categorical values in place with fitted ordinal indices, target included.

    Features use the shared ``encoder_object`` and versioned scalar keys; ``y``
    uses a separate ``__target__`` encoder with its existing string-label rules.
    Older feature artifacts retain their original string lookup. An unseen
    value becomes ``unknown_value`` unless ``handle_unknown`` is ``"error"``.
    Encoded output is ``float32``, not integer.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect feature and optional target encoders without changing lookup provenance."""
        fields = {"type", "columns", "encoder_object", "encoders", "categories_count"}
        if isinstance(raw, dict):
            fields.update({"category_key_version", "target_column"}.intersection(raw))
        if not local_state_fields(raw, "ordinal", fields, allow_empty=True):
            return raw
        uses_category_keys(raw)
        _validate_ordinal_features(raw)
        OrdinalEncoderApplier._validate_targets(raw)
        return raw

    @staticmethod
    def _validate_targets(raw: dict) -> None:
        """Keep embedded-target routing separate from selected feature encoders."""
        encoders = raw["encoders"]
        if type(encoders) is not dict or set(encoders) - {"__target__"}:
            raise ValueError("Unexpected fitted ordinal target encoders.")
        if "__target__" in encoders:
            _validate_ordinal_encoder(encoders["__target__"], 1)
        if "target_column" in raw and (
            not isinstance(raw["target_column"], str)
            or raw["target_column"] in raw["columns"]
            or "__target__" not in encoders
        ):
            raise ValueError("Fitted ordinal target routing disagrees with its encoders.")

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved ordinal lookups without enabling worker execution."""
        if engine not in ("pandas", "polars"):
            return None
        OrdinalEncoderApplier.validate_inference_state(state)
        # Legacy pandas datetime string formatting can depend on neighboring rows.
        legacy_pandas = (
            engine == "pandas" and state.get("columns") and not uses_category_keys(state)
        )
        context = "global" if legacy_pandas else "row"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific encode, forwarding ``(X, y)`` only when ``y`` exists."""
        return apply_target_encoder(
            X,
            y,
            params,
            {"polars": _ordinal_apply_polars, "pandas": _ordinal_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific encode, forwarding (X, y) only when y exists.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific encode, forwarding ``(X, y)`` only when ``y`` exists."""
    return apply_target_encoder(
        X,
        y,
        params,
        {"polars": _ordinal_apply_polars, "pandas": _ordinal_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe saved ordinal lookups without enabling worker execution.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved ordinal lookups without enabling worker execution."""
    if engine not in ("pandas", "polars"):
        return None
    OrdinalEncoderApplier.validate_inference_state(state)
    # Legacy pandas datetime string formatting can depend on neighboring rows.
    legacy_pandas = (
        engine == "pandas" and state.get("columns") and not uses_category_keys(state)
    )
    context = "global" if legacy_pandas else "row"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect feature and optional target encoders without changing lookup provenance.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect feature and optional target encoders without changing lookup provenance."""
    fields = {"type", "columns", "encoder_object", "encoders", "categories_count"}
    if isinstance(raw, dict):
        fields.update({"category_key_version", "target_column"}.intersection(raw))
    if not local_state_fields(raw, "ordinal", fields, allow_empty=True):
        return raw
    uses_category_keys(raw)
    _validate_ordinal_features(raw)
    OrdinalEncoderApplier._validate_targets(raw)
    return raw

OrdinalEncoderCalculator

Bases: BaseCalculator

Fit an ordinal encoder for the features and, when split out, for the target.

Picking no feature columns is not a no-op here — unlike its sibling encoders, this node still fits a target encoder when y is available. The target is encoded only when y exists and the configured target name is listed in columns but absent from X, i.e. it has already been split away. categories_order supplies an explicit ordering, and the target reads the last row of an n_features + 1 row table.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

@NodeRegistry.register("OrdinalEncoder", OrdinalEncoderApplier)
@node_meta(
    id="OrdinalEncoder",
    name="Ordinal Encoder",
    category="Preprocessing",
    description="Encodes categorical features as an integer array.",
    params={
        "columns": [],
        "handle_unknown": "use_encoded_value",
        "unknown_value": -1,
        "categories_order": "",
    },
    learns_from_data=True,
)
class OrdinalEncoderCalculator(BaseCalculator):
    """Fit an ordinal encoder for the features and, when split out, for the target.

    Picking no feature columns is *not* a no-op here — unlike its sibling
    encoders, this node still fits a target encoder when ``y`` is available.
    The target is encoded only when ``y`` exists and the configured target name
    is listed in ``columns`` but absent from ``X``, i.e. it has already been
    split away. ``categories_order`` supplies an explicit ordering, and the
    target reads the last row of an ``n_features + 1`` row table.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Pass the input schema through: encoded columns keep their name and position."""
        # Ordinal encoding replaces categorical values with ints in place;
        # column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> OrdinalArtifact:  # pylint: disable=arguments-differ
        """Fit the encoders on the frame's own engine, forwarding ``y`` when present."""
        return cast(
            OrdinalArtifact,
            fit_target_encoder(
                X,
                y,
                config,
                {"polars": _ordinal_fit_polars, "pandas": _ordinal_fit_pandas},
            ),
        )

fit(X, y, config)

Fit the encoders on the frame's own engine, forwarding y when present.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> OrdinalArtifact:  # pylint: disable=arguments-differ
    """Fit the encoders on the frame's own engine, forwarding ``y`` when present."""
    return cast(
        OrdinalArtifact,
        fit_target_encoder(
            X,
            y,
            config,
            {"polars": _ordinal_fit_polars, "pandas": _ordinal_fit_pandas},
        ),
    )

infer_output_schema(input_schema, config)

Pass the input schema through: encoded columns keep their name and position.

Source code in skyulf-core/skyulf/preprocessing/encoding/ordinal.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Pass the input schema through: encoded columns keep their name and position."""
    # Ordinal encoding replaces categorical values with ints in place;
    # column set is preserved.
    return input_schema

OversamplingApplier

Bases: BaseApplier

Grow the minority classes with the imblearn over-sampler named in the artifact.

Sees the training split only: :mod:.pipeline excludes the resampling nodes from apply_on_test/apply_on_validation, so synthetic rows can never reach a held-out split and inflate its metrics.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

class OversamplingApplier(BaseApplier):
    """Grow the minority classes with the imblearn over-sampler named in the artifact.

    Sees the training split only: :mod:`.pipeline` excludes the resampling nodes
    from ``apply_on_test``/``apply_on_validation``, so synthetic rows can never
    reach a held-out split and inflate its metrics.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved sampler configuration without fitting optional estimators."""
        return _validate_resampling_state(raw, "oversampling", "smote")

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Require the full class distribution; mixed SMOTETomek effects stay unknown."""
        if engine not in ("pandas", "polars"):
            return None
        OversamplingApplier.validate_inference_state(state)
        if not _over_context_known(state):
            return None
        return ExecutionCapability(engine, "apply", "local", "expand", "global")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Resample ``(X, y)`` with the configured over-sampler.

        ``y`` is lifted out of ``X`` via ``target_column`` when not supplied
        separately; with no usable target the data passes through untouched, as
        it does for a ``method`` no builder recognizes.

        Raises:
            ValueError: If any feature column is non-numeric, which imblearn
                cannot resample. Encode first.
        """
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {
                "polars": lambda Xi, yi, p: _resample_polars(
                    Xi, yi, p, _build_oversampler, "smote"
                ),
                "pandas": lambda Xi, yi, p: _resample_pandas(
                    Xi, yi, p, _build_oversampler, "smote"
                ),
            },
        )

apply(X, y, params)

Resample (X, y) with the configured over-sampler.

y is lifted out of X via target_column when not supplied separately; with no usable target the data passes through untouched, as it does for a method no builder recognizes.

Raises:

Type	Description
ValueError	If any feature column is non-numeric, which imblearn cannot resample. Encode first.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Resample ``(X, y)`` with the configured over-sampler.

    ``y`` is lifted out of ``X`` via ``target_column`` when not supplied
    separately; with no usable target the data passes through untouched, as
    it does for a ``method`` no builder recognizes.

    Raises:
        ValueError: If any feature column is non-numeric, which imblearn
            cannot resample. Encode first.
    """
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {
            "polars": lambda Xi, yi, p: _resample_polars(
                Xi, yi, p, _build_oversampler, "smote"
            ),
            "pandas": lambda Xi, yi, p: _resample_pandas(
                Xi, yi, p, _build_oversampler, "smote"
            ),
        },
    )

inference_capability(state, *, engine) staticmethod

Require the full class distribution; mixed SMOTETomek effects stay unknown.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Require the full class distribution; mixed SMOTETomek effects stay unknown."""
    if engine not in ("pandas", "polars"):
        return None
    OversamplingApplier.validate_inference_state(state)
    if not _over_context_known(state):
        return None
    return ExecutionCapability(engine, "apply", "local", "expand", "global")

validate_inference_state(raw) staticmethod

Inspect the saved sampler configuration without fitting optional estimators.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved sampler configuration without fitting optional estimators."""
    return _validate_resampling_state(raw, "oversampling", "smote")

OversamplingCalculator

Bases: BaseCalculator

Configure class balancing by oversampling the minority class.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@NodeRegistry.register("Oversampling", OversamplingApplier)
@node_meta(
    id="Oversampling",
    name="Oversampling",
    category="Preprocessing",
    description="Resample dataset to balance classes by oversampling minority class.",
    params={"method": "smote", "target_column": "target", "sampling_strategy": "auto"},
    learns_from_data=True,
)
class OversamplingCalculator(BaseCalculator):
    """Configure class balancing by oversampling the minority class."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Resampling changes row counts only; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> OversamplingArtifact:  # pylint: disable=arguments-differ
        """Snapshot the sampler settings from ``config`` into the artifact.

        Neither ``_X`` nor ``_y`` is read: the artifact is pure configuration,
        and the class imbalance it corrects is only observed when the applier
        runs the sampler on the training split.
        """
        return {
            "type": "oversampling",
            "synthetic_weight": config.get("synthetic_weight"),
            "method": config.get("method", "smote"),
            "target_column": config.get("target_column"),
            "sampling_strategy": config.get("sampling_strategy", "auto"),
            "random_state": config.get("random_state", 42),
            "k_neighbors": config.get("k_neighbors", 5),
            "m_neighbors": config.get("m_neighbors", 10),
            "kind": config.get("kind", "borderline-1"),
            "svm_estimator": config.get("svm_estimator"),
            "out_step": config.get("out_step", 0.5),
            "kmeans_estimator": config.get("kmeans_estimator"),
            "cluster_balance_threshold": config.get("cluster_balance_threshold", 0.1),
            "density_exponent": config.get("density_exponent", "auto"),
            "n_jobs": config.get("n_jobs", -1),
        }

fit(_X, _y, config)

Snapshot the sampler settings from config into the artifact.

Neither _X nor _y is read: the artifact is pure configuration, and the class imbalance it corrects is only observed when the applier runs the sampler on the training split.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@fit_method
def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> OversamplingArtifact:  # pylint: disable=arguments-differ
    """Snapshot the sampler settings from ``config`` into the artifact.

    Neither ``_X`` nor ``_y`` is read: the artifact is pure configuration,
    and the class imbalance it corrects is only observed when the applier
    runs the sampler on the training split.
    """
    return {
        "type": "oversampling",
        "synthetic_weight": config.get("synthetic_weight"),
        "method": config.get("method", "smote"),
        "target_column": config.get("target_column"),
        "sampling_strategy": config.get("sampling_strategy", "auto"),
        "random_state": config.get("random_state", 42),
        "k_neighbors": config.get("k_neighbors", 5),
        "m_neighbors": config.get("m_neighbors", 10),
        "kind": config.get("kind", "borderline-1"),
        "svm_estimator": config.get("svm_estimator"),
        "out_step": config.get("out_step", 0.5),
        "kmeans_estimator": config.get("kmeans_estimator"),
        "cluster_balance_threshold": config.get("cluster_balance_threshold", 0.1),
        "density_exponent": config.get("density_exponent", "auto"),
        "n_jobs": config.get("n_jobs", -1),
    }

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Resampling changes row counts only; column set is preserved.
    return input_schema

PolynomialFeaturesApplier

Bases: BaseApplier

Append the polynomial expansion described by the artifact.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

class PolynomialFeaturesApplier(BaseApplier):
    """Append the polynomial expansion described by the artifact."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved expansion configuration without rebuilding or fitting its terms."""
        if not local_state_fields(
            raw,
            "polynomial_features",
            {
                "type",
                "columns",
                "degree",
                "interaction_only",
                "include_bias",
                "include_input_features",
                "output_prefix",
                "feature_names",
            },
            allow_empty=True,
        ):
            return raw
        _validate_polynomial_names(raw["columns"])
        if len(set(raw["columns"])) != len(raw["columns"]):
            raise ValueError("Fitted polynomial columns must be unique.")
        for flag in ("interaction_only", "include_bias", "include_input_features"):
            local_boolean(raw[flag], flag)
        _validate_polynomial_degree(raw["degree"], raw["include_bias"])
        if not isinstance(raw["output_prefix"], str):
            raise ValueError("Fitted polynomial prefix must be a string.")
        _validate_polynomial_names(raw["feature_names"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local polynomial arithmetic without granting worker execution."""
        if engine not in ("pandas", "polars"):
            return None
        PolynomialFeaturesApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Recompute the expansion from ``X`` and append it as new columns.

        ``PolynomialFeatures`` is refit here rather than persisted, so the
        artifact only carries configuration. Artifact columns absent from ``X``
        are filtered out first, and the default ``include_input_features=False``
        drops the degree-1 terms so the originals are not duplicated. ``X`` is
        returned unchanged when nothing survives. Emitted names must be unique
        and must not collide with existing columns.
        """
        return apply_dual_engine(
            X, params, {"polars": _polynomial_apply_polars, "pandas": _polynomial_apply_pandas}
        )

apply(X, _y, params)

Recompute the expansion from X and append it as new columns.

PolynomialFeatures is refit here rather than persisted, so the artifact only carries configuration. Artifact columns absent from X are filtered out first, and the default include_input_features=False drops the degree-1 terms so the originals are not duplicated. X is returned unchanged when nothing survives. Emitted names must be unique and must not collide with existing columns.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Recompute the expansion from ``X`` and append it as new columns.

    ``PolynomialFeatures`` is refit here rather than persisted, so the
    artifact only carries configuration. Artifact columns absent from ``X``
    are filtered out first, and the default ``include_input_features=False``
    drops the degree-1 terms so the originals are not duplicated. ``X`` is
    returned unchanged when nothing survives. Emitted names must be unique
    and must not collide with existing columns.
    """
    return apply_dual_engine(
        X, params, {"polars": _polynomial_apply_polars, "pandas": _polynomial_apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local polynomial arithmetic without granting worker execution.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local polynomial arithmetic without granting worker execution."""
    if engine not in ("pandas", "polars"):
        return None
    PolynomialFeaturesApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved expansion configuration without rebuilding or fitting its terms.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved expansion configuration without rebuilding or fitting its terms."""
    if not local_state_fields(
        raw,
        "polynomial_features",
        {
            "type",
            "columns",
            "degree",
            "interaction_only",
            "include_bias",
            "include_input_features",
            "output_prefix",
            "feature_names",
        },
        allow_empty=True,
    ):
        return raw
    _validate_polynomial_names(raw["columns"])
    if len(set(raw["columns"])) != len(raw["columns"]):
        raise ValueError("Fitted polynomial columns must be unique.")
    for flag in ("interaction_only", "include_bias", "include_input_features"):
        local_boolean(raw[flag], flag)
    _validate_polynomial_degree(raw["degree"], raw["include_bias"])
    if not isinstance(raw["output_prefix"], str):
        raise ValueError("Fitted polynomial prefix must be a string.")
    _validate_polynomial_names(raw["feature_names"])
    return raw

PolynomialFeaturesCalculator

Bases: BaseCalculator

Resolve the polynomial configuration and the names it will emit.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

@NodeRegistry.register("PolynomialFeatures", PolynomialFeaturesApplier)
@NodeRegistry.register("PolynomialFeaturesNode", PolynomialFeaturesApplier)
@node_meta(
    id="PolynomialFeatures",
    name="Polynomial Features",
    category="Feature Engineering",
    description="Generate polynomial and interaction features.",
    params={"degree": 2, "interaction_only": False, "include_bias": False},
    # Automatic column discovery inspects values; fixed selections are exempted.
    learns_from_data=True,
)
class PolynomialFeaturesCalculator(BaseCalculator):
    """Resolve the polynomial configuration and the names it will emit."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> PolynomialFeaturesArtifact:  # pylint: disable=arguments-differ
        """Record the expansion settings and the feature names they produce.

        ``columns`` falls back to auto-detected numeric columns when
        ``auto_detect`` is set. ``PolynomialFeatures`` is fitted here purely to
        derive ``feature_names`` — the expansion itself is recomputed at apply
        time. Returns an empty artifact, a no-op passthrough, when no column
        resolves. Ambiguous generated names raise ``ValueError`` before replay.
        """
        cols = list(config.get("columns", []))
        if not cols and config.get("auto_detect", False):
            # detect_numeric_columns dispatches natively on Polars frames too,
            # so this doesn't require converting the full frame first.
            cols = detect_numeric_columns(X)
        X_np, cols = select_then_to_numpy(X, cols)
        if not cols:
            return cast(PolynomialFeaturesArtifact, {})

        degree = config.get("degree", 2)
        interaction_only = config.get("interaction_only", False)
        include_bias = config.get("include_bias", False)

        poly = PolynomialFeatures(
            degree=degree, interaction_only=interaction_only, include_bias=include_bias
        )
        poly.fit(X_np)
        _, names = _polynomial_names(poly, cols, config)
        _validate_generated_names(names, list(X.columns), "PolynomialFeatures")
        return cast(
            PolynomialFeaturesArtifact,
            {
                "type": "polynomial_features",
                "columns": cols,
                "degree": degree,
                "interaction_only": interaction_only,
                "include_bias": include_bias,
                "include_input_features": config.get("include_input_features", False),
                "output_prefix": config.get("output_prefix", "poly"),
                "feature_names": poly.get_feature_names_out(cols).tolist(),
            },
        )

fit(X, _y, config)

Record the expansion settings and the feature names they produce.

columns falls back to auto-detected numeric columns when auto_detect is set. PolynomialFeatures is fitted here purely to derive feature_names — the expansion itself is recomputed at apply time. Returns an empty artifact, a no-op passthrough, when no column resolves. Ambiguous generated names raise ValueError before replay.

Source code in skyulf-core/skyulf/preprocessing/feature_generation/polynomial.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> PolynomialFeaturesArtifact:  # pylint: disable=arguments-differ
    """Record the expansion settings and the feature names they produce.

    ``columns`` falls back to auto-detected numeric columns when
    ``auto_detect`` is set. ``PolynomialFeatures`` is fitted here purely to
    derive ``feature_names`` — the expansion itself is recomputed at apply
    time. Returns an empty artifact, a no-op passthrough, when no column
    resolves. Ambiguous generated names raise ``ValueError`` before replay.
    """
    cols = list(config.get("columns", []))
    if not cols and config.get("auto_detect", False):
        # detect_numeric_columns dispatches natively on Polars frames too,
        # so this doesn't require converting the full frame first.
        cols = detect_numeric_columns(X)
    X_np, cols = select_then_to_numpy(X, cols)
    if not cols:
        return cast(PolynomialFeaturesArtifact, {})

    degree = config.get("degree", 2)
    interaction_only = config.get("interaction_only", False)
    include_bias = config.get("include_bias", False)

    poly = PolynomialFeatures(
        degree=degree, interaction_only=interaction_only, include_bias=include_bias
    )
    poly.fit(X_np)
    _, names = _polynomial_names(poly, cols, config)
    _validate_generated_names(names, list(X.columns), "PolynomialFeatures")
    return cast(
        PolynomialFeaturesArtifact,
        {
            "type": "polynomial_features",
            "columns": cols,
            "degree": degree,
            "interaction_only": interaction_only,
            "include_bias": include_bias,
            "include_input_features": config.get("include_input_features", False),
            "output_prefix": config.get("output_prefix", "poly"),
            "feature_names": poly.get_feature_names_out(cols).tolist(),
        },
    )

PowerTransformerApplier

Bases: BaseApplier

Apply the fitted Box-Cox / Yeo-Johnson transform to the selected columns.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

class PowerTransformerApplier(BaseApplier):
    """Apply the fitted Box-Cox / Yeo-Johnson transform to the selected columns."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved lambdas and scaler without normalizing or relearning them."""
        if local_state_fields(
            raw,
            "power_transformer",
            {"type", "columns", "method", "standardize", "lambdas", "scaler_params"},
            allow_empty=True,
        ):
            columns = raw["columns"]
            if type(columns) is not list or any(not isinstance(col, str) for col in columns):
                raise ValueError("Fitted power columns must be a list of strings.")
            validate_power_parameters(raw, len(columns))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Expose whole-request fallback when any value prevents a power transform."""
        if engine not in ("pandas", "polars"):
            return None
        PowerTransformerApplier.validate_inference_state(state)
        context = "global" if state.get("columns") else "row"
        return ExecutionCapability(engine, "apply", "local", "preserve", context)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Rebuild the transformer from stored lambdas and transform ``X``; ``y`` untouched.

        Fails open, like every applier in ``preprocessing/transformations/``: a
        failed transform is logged at ERROR with its traceback and leaves the data
        unchanged, so a degraded frame beats a hard failure at inference time.
        Every path that leaves data untransformed logs — including the one that
        used not to, a frame missing columns the transformer was fitted on, which
        is train/serve skew rather than a no-op and is warned about by name.
        """
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        if params.get("lambdas") is None:
            return X, _y
        valid_cols = _fitted_columns_present(X, cols)
        if not valid_cols:
            return X, _y

        try:
            X_vals = X.select(valid_cols).to_numpy()
            X_trans = _power_transform_array(X_vals, params, cols, valid_cols)
            series = [pl.Series(name, X_trans[:, i]) for i, name in enumerate(valid_cols)]
            return X.with_columns(series), _y
        except Exception:  # noqa: BLE001 - transform failure is logged; frame left unchanged
            logger.exception("PowerTransformer (Polars) application failed")
            return X, _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        if params.get("lambdas") is None:
            return X, _y
        valid_cols = _fitted_columns_present(X, cols)
        if not valid_cols:
            return X, _y

        try:
            # Computed before the copy so a failure hands back the caller's frame
            # rather than one whose columns were already cast to float64, matching
            # what the polars path returns.
            subset = decimal_columns_to_float(X[valid_cols], valid_cols)
            X_vals = subset.to_numpy(dtype=np.float64, na_value=np.nan)
            X_trans = np.asarray(_power_transform_array(X_vals, params, cols, valid_cols))
            df_out = X.copy()
            # Replace selected columns so nullable integers and Decimal objects
            # can receive float results without incompatible-dtype assignment.
            df_out[valid_cols] = X_trans
        except Exception:  # noqa: BLE001 - transform failure is logged; frame left unchanged
            logger.exception("PowerTransformer (Pandas) application failed")
            return X, _y
        return df_out, _y

apply(X, _y, params)

Rebuild the transformer from stored lambdas and transform X; y untouched.

Fails open, like every applier in preprocessing/transformations/: a failed transform is logged at ERROR with its traceback and leaves the data unchanged, so a degraded frame beats a hard failure at inference time. Every path that leaves data untransformed logs — including the one that used not to, a frame missing columns the transformer was fitted on, which is train/serve skew rather than a no-op and is warned about by name.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Rebuild the transformer from stored lambdas and transform ``X``; ``y`` untouched.

    Fails open, like every applier in ``preprocessing/transformations/``: a
    failed transform is logged at ERROR with its traceback and leaves the data
    unchanged, so a degraded frame beats a hard failure at inference time.
    Every path that leaves data untransformed logs — including the one that
    used not to, a frame missing columns the transformer was fitted on, which
    is train/serve skew rather than a no-op and is warned about by name.
    """
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Expose whole-request fallback when any value prevents a power transform.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Expose whole-request fallback when any value prevents a power transform."""
    if engine not in ("pandas", "polars"):
        return None
    PowerTransformerApplier.validate_inference_state(state)
    context = "global" if state.get("columns") else "row"
    return ExecutionCapability(engine, "apply", "local", "preserve", context)

validate_inference_state(raw) staticmethod

Inspect the saved lambdas and scaler without normalizing or relearning them.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved lambdas and scaler without normalizing or relearning them."""
    if local_state_fields(
        raw,
        "power_transformer",
        {"type", "columns", "method", "standardize", "lambdas", "scaler_params"},
        allow_empty=True,
    ):
        columns = raw["columns"]
        if type(columns) is not list or any(not isinstance(col, str) for col in columns):
            raise ValueError("Fitted power columns must be a list of strings.")
        validate_power_parameters(raw, len(columns))
    return raw

PowerTransformerCalculator

Bases: BaseCalculator

Fit one shared PowerTransformer across the selected columns.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

@NodeRegistry.register("PowerTransformer", PowerTransformerApplier)
@node_meta(
    id="PowerTransformer",
    name="Power Transformer",
    category="Preprocessing",
    description="Apply a power transform featurewise to make data more Gaussian-like.",
    params={"method": "yeo-johnson", "standardize": True, "columns": []},
    learns_from_data=True,
)
class PowerTransformerCalculator(BaseCalculator):
    """Fit one shared PowerTransformer across the selected columns."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with transformed columns promoted to ``float64``."""
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> PowerTransformerArtifact:  # pylint: disable=arguments-differ
        """Fit per-column lambdas, dropping columns Box-Cox cannot accept."""
        if user_picked_no_columns(config):
            return {}

        method = config.get("method", "yeo-johnson")
        standardize = config.get("standardize", True)
        # TODO(pandas-removal): PowerTransformer.fit accepts numpy directly,
        # but _filter_power_columns() uses Pandas column-wise boolean
        # indexing (`X_pd[c] <= 0`) to drop non-positive columns for box-cox.
        # That check is easy to port to numpy (X_np[:, i] <= 0), but do it
        # together with a matching rewrite of _filter_power_columns to take
        # an ndarray, so both stay in sync.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        valid_cols = _filter_power_columns(X_pd, cols, method)
        if not valid_cols:
            return {}

        transformer = PowerTransformer(method=method, standardize=standardize)
        transformer.fit(X_pd[valid_cols])

        return {
            "type": "power_transformer",
            "lambdas": transformer.lambdas_.tolist(),
            "method": method,
            "standardize": standardize,
            "columns": valid_cols,
            "scaler_params": _extract_scaler_params(transformer, standardize),
        }

fit(X, _y, config)

Fit per-column lambdas, dropping columns Box-Cox cannot accept.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> PowerTransformerArtifact:  # pylint: disable=arguments-differ
    """Fit per-column lambdas, dropping columns Box-Cox cannot accept."""
    if user_picked_no_columns(config):
        return {}

    method = config.get("method", "yeo-johnson")
    standardize = config.get("standardize", True)
    # TODO(pandas-removal): PowerTransformer.fit accepts numpy directly,
    # but _filter_power_columns() uses Pandas column-wise boolean
    # indexing (`X_pd[c] <= 0`) to drop non-positive columns for box-cox.
    # That check is easy to port to numpy (X_np[:, i] <= 0), but do it
    # together with a matching rewrite of _filter_power_columns to take
    # an ndarray, so both stay in sync.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    valid_cols = _filter_power_columns(X_pd, cols, method)
    if not valid_cols:
        return {}

    transformer = PowerTransformer(method=method, standardize=standardize)
    transformer.fit(X_pd[valid_cols])

    return {
        "type": "power_transformer",
        "lambdas": transformer.lambdas_.tolist(),
        "method": method,
        "standardize": standardize,
        "columns": valid_cols,
        "scaler_params": _extract_scaler_params(transformer, standardize),
    }

infer_output_schema(input_schema, config)

Return a schema with transformed columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/transformations/power.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with transformed columns promoted to ``float64``."""
    return promote_configured_columns_to_float64(input_schema, config)

RobustScalerApplier

Bases: BaseApplier

Center on the fitted median and scale by the fitted quantile spread.

Honors with_centering/with_scaling; y is never modified.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

class RobustScalerApplier(BaseApplier):
    """Center on the fitted median and scale by the fitted quantile spread.

    Honors ``with_centering``/``with_scaling``; ``y`` is never modified.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved local flags and statistics without recalculating quantiles."""
        if not local_state_fields(
            raw,
            "robust_scaler",
            {
                "type",
                "columns",
                "center",
                "scale",
                "quantile_range",
                "with_centering",
                "with_scaling",
            },
            allow_empty=True,
        ):
            return raw
        columns = _columns(raw["columns"])
        if type(raw["quantile_range"]) not in (list, tuple):
            raise ValueError("Fitted robust scaler quantiles must be a pair of bounds.")
        validate_scaling_range(raw["quantile_range"], "quantile_range")
        _validate_robust_statistic(raw, "with_centering", "center", len(columns))
        _validate_robust_statistic(raw, "with_scaling", "scale", len(columns))
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local saved apply context without granting worker execution."""
        if engine not in ("pandas", "polars"):
            return None
        RobustScalerApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Apply the fitted robust center/scale to ``X`` on the active engine."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        center = params.get("center")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        if not valid:
            return X, _y

        with_centering = params.get("with_centering", True)
        with_scaling = params.get("with_scaling", True)
        if not any((with_centering, with_scaling)):
            return X, _y

        exprs = []
        for col_name in valid:
            i = cols.index(col_name)
            column = pl.col(col_name) if isinstance(X, pl.LazyFrame) else X.get_column(col_name)
            e = column.cast(pl.Float64)
            if with_centering and center is not None:
                e = e - center[i]
            if with_scaling and scale is not None:
                s = scale[i]
                e = divide_polars_column(X, e, s)
            exprs.append(e.alias(col_name))
        return X.with_columns(exprs), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        center = params.get("center")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        if not valid:
            return X, _y

        if not (params.get("with_centering", True) or params.get("with_scaling", True)):
            return X, _y

        X_out = X.copy()
        col_indices = [cols.index(c) for c in valid]
        subset = decimal_columns_to_float(X_out[valid], valid)
        vals = subset.to_numpy(dtype=np.float64, na_value=np.nan)

        if params.get("with_centering", True) and center is not None:
            vals = vals - np.array(center)[col_indices]
        if params.get("with_scaling", True) and scale is not None:
            vals = vals / safe_scale(np.array(scale)[col_indices].copy())

        X_out[valid] = vals
        return X_out, _y

apply(X, _y, params)

Apply the fitted robust center/scale to X on the active engine.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Apply the fitted robust center/scale to ``X`` on the active engine."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local saved apply context without granting worker execution.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local saved apply context without granting worker execution."""
    if engine not in ("pandas", "polars"):
        return None
    RobustScalerApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved local flags and statistics without recalculating quantiles.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved local flags and statistics without recalculating quantiles."""
    if not local_state_fields(
        raw,
        "robust_scaler",
        {
            "type",
            "columns",
            "center",
            "scale",
            "quantile_range",
            "with_centering",
            "with_scaling",
        },
        allow_empty=True,
    ):
        return raw
    columns = _columns(raw["columns"])
    if type(raw["quantile_range"]) not in (list, tuple):
        raise ValueError("Fitted robust scaler quantiles must be a pair of bounds.")
    validate_scaling_range(raw["quantile_range"], "quantile_range")
    _validate_robust_statistic(raw, "with_centering", "center", len(columns))
    _validate_robust_statistic(raw, "with_scaling", "scale", len(columns))
    return raw

RobustScalerCalculator

Bases: BaseCalculator

Fit outlier-robust median/quantile-range statistics via sklearn's RobustScaler.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

@NodeRegistry.register("RobustScaler", RobustScalerApplier)
@node_meta(
    id="RobustScaler",
    name="Robust Scaler",
    category="Preprocessing",
    description="Scale features using statistics that are robust to outliers.",
    params={
        "quantile_range": [25.0, 75.0],
        "with_centering": True,
        "with_scaling": True,
        "columns": [],
    },
    learns_from_data=True,
)
class RobustScalerCalculator(BaseCalculator):
    """Fit outlier-robust median/quantile-range statistics via sklearn's ``RobustScaler``."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with transformed columns promoted to ``float64``."""
        if not (config.get("with_centering", True) or config.get("with_scaling", True)):
            return input_schema
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> RobustScalerArtifact:  # pylint: disable=arguments-differ
        """Dispatch the fit to the active engine and return the robust-statistics artifact."""
        if user_picked_no_columns(config):
            return cast(RobustScalerArtifact, {})
        return cast(
            RobustScalerArtifact,
            fit_dual_engine(X, config, {"polars": self._fit_polars, "pandas": self._fit_pandas}),
        )

    @staticmethod
    def _fit_polars(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_polars(X, config)
        if not cols:
            return {}
        return _fit_robust(X_subset, cols, config)

    @staticmethod
    def _fit_pandas(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_pandas(X, config)
        if not cols:
            return {}
        return _fit_robust(X_subset, cols, config)

fit(X, _y, config)

Dispatch the fit to the active engine and return the robust-statistics artifact.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> RobustScalerArtifact:  # pylint: disable=arguments-differ
    """Dispatch the fit to the active engine and return the robust-statistics artifact."""
    if user_picked_no_columns(config):
        return cast(RobustScalerArtifact, {})
    return cast(
        RobustScalerArtifact,
        fit_dual_engine(X, config, {"polars": self._fit_polars, "pandas": self._fit_pandas}),
    )

infer_output_schema(input_schema, config)

Return a schema with transformed columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/scaling/robust.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with transformed columns promoted to ``float64``."""
    if not (config.get("with_centering", True) or config.get("with_scaling", True)):
        return input_schema
    return promote_configured_columns_to_float64(input_schema, config)

RollingAggregateApplier

Bases: BaseApplier

Append rolling-window aggregate columns for the configured columns.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

class RollingAggregateApplier(BaseApplier):
    """Append rolling-window aggregate columns for the configured columns."""

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe request-window dependence independently of fitted history seeds."""
        if engine not in ("pandas", "polars"):
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "window")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Compute the rolling aggregates on the active engine; ``y`` passes through."""
        return apply_dual_engine(
            (X, _y) if _y is not None else X,
            params,
            {"polars": _apply_polars, "pandas": _apply_pandas},
        )

apply(X, _y, params)

Compute the rolling aggregates on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Compute the rolling aggregates on the active engine; ``y`` passes through."""
    return apply_dual_engine(
        (X, _y) if _y is not None else X,
        params,
        {"polars": _apply_polars, "pandas": _apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe request-window dependence independently of fitted history seeds.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe request-window dependence independently of fitted history seeds."""
    if engine not in ("pandas", "polars"):
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "window")

RollingAggregateCalculator

Bases: BaseCalculator

Save rolling configuration and optional bounded training history.

Apply includes the current row; carry mode also sees saved earlier rows. Rolling the known target is rejected; other inputs must be available at prediction time.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

@NodeRegistry.register("RollingAggregate", RollingAggregateApplier)
@node_meta(
    id="RollingAggregate",
    name="Rolling Aggregate",
    category="Preprocessing",
    description="Rolling-window aggregates (mean/sum/min/max/std/median) for time series.",
    params={
        "columns": [],
        "window": 3,
        "aggregations": ["mean"],
        "min_periods": 1,
        "group_by": None,
        "sort_by": None,
    },
    tags=["time-series"],
    learns_from_data=False,
)
class RollingAggregateCalculator(BaseCalculator):
    """Save rolling configuration and optional bounded training history.

    Apply includes the current row; carry mode also sees saved earlier rows. Rolling
    the known target is rejected; other inputs must be available at prediction time.
    """

    def fit(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        config: dict[str, Any],
    ) -> RollingAggregateArtifact:
        """Record the window, recognized aggregations, and sort/group settings."""
        validate_temporal_target(
            "RollingAggregate", config, target_column=config.get("target_column")
        )
        params: dict[str, Any] = {
            "type": "rolling_aggregate",
            "columns": config.get("columns", []),
            "window": int(config.get("window", 3)),
            "aggregations": coerce_aggregations(config.get("aggregations", ["mean"])),
            "min_periods": int(config.get("min_periods", 1)),
            "group_by": config.get("group_by"),
            "sort_by": config.get("sort_by"),
        }
        return cast(RollingAggregateArtifact, fit_history(df, config, params, params["window"] - 1))

    def fit_transform_train(self, df: Any, config: dict[str, Any]) -> tuple[Any, Any]:
        """Save the training tail without using it as context for the training rows."""
        params = self.fit(df, config)
        return params, RollingAggregateApplier().apply(
            df, dict(params) | {"_history_training": True}
        )

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema | None:
        """Add one float64 ``{col}_roll_{agg}_{window}`` column per (column, aggregation)."""
        # Rolling outputs are float64 regardless of source dtype.
        cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
        aggs = coerce_aggregations(config.get("aggregations", ["mean"]))
        window = int(config.get("window", 3))
        schema = input_schema
        for col in cols:
            for agg in aggs:
                schema = schema.add(_roll_name(col, agg, window), "float64")
        return schema

fit(df, config)

Record the window, recognized aggregations, and sort/group settings.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

def fit(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    config: dict[str, Any],
) -> RollingAggregateArtifact:
    """Record the window, recognized aggregations, and sort/group settings."""
    validate_temporal_target(
        "RollingAggregate", config, target_column=config.get("target_column")
    )
    params: dict[str, Any] = {
        "type": "rolling_aggregate",
        "columns": config.get("columns", []),
        "window": int(config.get("window", 3)),
        "aggregations": coerce_aggregations(config.get("aggregations", ["mean"])),
        "min_periods": int(config.get("min_periods", 1)),
        "group_by": config.get("group_by"),
        "sort_by": config.get("sort_by"),
    }
    return cast(RollingAggregateArtifact, fit_history(df, config, params, params["window"] - 1))

fit_transform_train(df, config)

Save the training tail without using it as context for the training rows.

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

def fit_transform_train(self, df: Any, config: dict[str, Any]) -> tuple[Any, Any]:
    """Save the training tail without using it as context for the training rows."""
    params = self.fit(df, config)
    return params, RollingAggregateApplier().apply(
        df, dict(params) | {"_history_training": True}
    )

infer_output_schema(input_schema, config)

Add one float64 {col}_roll_{agg}_{window} column per (column, aggregation).

Source code in skyulf-core/skyulf/preprocessing/time_series/rolling.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema | None:
    """Add one float64 ``{col}_roll_{agg}_{window}`` column per (column, aggregation)."""
    # Rolling outputs are float64 regardless of source dtype.
    cols = filter_existing_columns(config.get("columns", []), input_schema.column_list())
    aggs = coerce_aggregations(config.get("aggregations", ["mean"]))
    window = int(config.get("window", 3))
    schema = input_schema
    for col in cols:
        for agg in aggs:
            schema = schema.add(_roll_name(col, agg, window), "float64")
    return schema

SchemaMismatchError

Bases: ValueError

Raised when an actual frame schema violates an expected SkyulfSchema.

Carries structured details so callers can render a precise message instead of a generic KeyError deep inside a transformer:

Attributes:

Name	Type	Description
missing		Expected columns absent from the actual frame.
unexpected		Actual columns not present in the expected schema.
dtype_mismatches		{column: (expected_dtype, actual_dtype)} for shared columns whose dtype labels differ (only when dtype checking is requested).
order_mismatch		True when the shared columns appear in a different relative order than expected (only when order checking is requested).

Source code in skyulf-core/skyulf/core/schema.py

class SchemaMismatchError(ValueError):
    """Raised when an actual frame schema violates an expected ``SkyulfSchema``.

    Carries structured details so callers can render a precise message instead
    of a generic ``KeyError`` deep inside a transformer:

    Attributes:
        missing: Expected columns absent from the actual frame.
        unexpected: Actual columns not present in the expected schema.
        dtype_mismatches: ``{column: (expected_dtype, actual_dtype)}`` for
            shared columns whose dtype labels differ (only when dtype checking
            is requested).
        order_mismatch: ``True`` when the shared columns appear in a different
            relative order than expected (only when order checking is requested).
    """

    def __init__(
        self,
        message: str,
        *,
        missing: list[str] | None = None,
        unexpected: list[str] | None = None,
        dtype_mismatches: dict[str, tuple[str, str]] | None = None,
        order_mismatch: bool = False,
    ) -> None:
        """Set the ``ValueError`` message and capture the structured diff fields as attributes."""
        super().__init__(message)
        self.missing = missing or []
        self.unexpected = unexpected or []
        self.dtype_mismatches = dtype_mismatches or {}
        self.order_mismatch = order_mismatch

__init__(message, *, missing=None, unexpected=None, dtype_mismatches=None, order_mismatch=False)

Set the ValueError message and capture the structured diff fields as attributes.

Source code in skyulf-core/skyulf/core/schema.py

def __init__(
    self,
    message: str,
    *,
    missing: list[str] | None = None,
    unexpected: list[str] | None = None,
    dtype_mismatches: dict[str, tuple[str, str]] | None = None,
    order_mismatch: bool = False,
) -> None:
    """Set the ``ValueError`` message and capture the structured diff fields as attributes."""
    super().__init__(message)
    self.missing = missing or []
    self.unexpected = unexpected or []
    self.dtype_mismatches = dtype_mismatches or {}
    self.order_mismatch = order_mismatch

SentenceEmbedderApplier

Bases: BaseApplier

Encode the joined text of the configured columns into embedding_dim float columns.

All configured columns are concatenated into one corpus, so a multi-column selection yields a single embedding whose name prefix joins the source names. Width is set by the model, not by config, and is recorded in the artifact at fit time. New artifacts cache loaded CPU encoders by their immutable saved bytes, so models sharing a display name cannot select each other's weights. normalize=True (the default) yields unit-length vectors ready for cosine similarity. Because the optional sentence-transformers import is lazy, a missing extra surfaces here as an ImportError carrying the install hint rather than at skyulf import time. The polars path keeps the frame native around the encode, but the encoding itself always runs outside the dataframe engine.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

class SentenceEmbedderApplier(BaseApplier):
    """Encode the joined text of the configured columns into ``embedding_dim`` float columns.

    All configured columns are concatenated into one corpus, so a multi-column
    selection yields a single embedding whose name prefix joins the source names.
    Width is set by the model, not by config, and is recorded in the artifact at
    fit time. New artifacts cache loaded CPU encoders by their immutable saved
    bytes, so models sharing a display name cannot select each other's weights.
    ``normalize=True`` (the default)
    yields unit-length vectors ready for cosine similarity. Because the optional
    ``sentence-transformers`` import is lazy, a missing extra surfaces here as an
    ``ImportError`` carrying the install hint rather than at skyulf import time.
    The polars path keeps the frame native around the encode, but the encoding
    itself always runs outside the dataframe engine.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved schema and asset identity without loading or running the encoder."""
        if type(raw) is not dict:
            raise ValueError("Local fitted state must be a dictionary.")
        fields = {
            "type",
            "columns",
            "model_name",
            "embedding_dim",
            "normalize",
            "output_columns",
            "drop_original",
        }
        assets = {"model_snapshot", "model_sha256", "model_requirements"}
        fields |= assets if assets.intersection(raw) else set()
        if not local_state_fields(raw, "sentence_embedder", fields, allow_empty=True):
            return raw
        local_scalar(raw["normalize"], "normalize")
        local_scalar(raw["drop_original"], "drop_original")
        _validate_text_layout({**raw, "drop_original": bool(raw["drop_original"])})
        if not isinstance(raw["model_name"], str) or not raw["model_name"]:
            raise ValueError("Fitted sentence model name must be nonempty text.")
        width = raw["embedding_dim"]
        if type(width) is not int or width <= 0 or len(raw["output_columns"]) != width:
            raise ValueError("Fitted sentence embedding width disagrees with its columns.")
        if "model_snapshot" in raw:
            _validate_snapshot(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe packaged row-local encoding; name-only legacy state stays unknown."""
        if engine not in ("pandas", "polars"):
            return None
        SentenceEmbedderApplier.validate_inference_state(state)
        if state and "model_snapshot" not in state:
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
        return apply_text_dual_engine(X, params, _embed_apply_pandas, _embed_apply_polars)

apply(X, _y, params)

Dispatch through the text-specific dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
    return apply_text_dual_engine(X, params, _embed_apply_pandas, _embed_apply_polars)

inference_capability(state, *, engine) staticmethod

Describe packaged row-local encoding; name-only legacy state stays unknown.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe packaged row-local encoding; name-only legacy state stays unknown."""
    if engine not in ("pandas", "polars"):
        return None
    SentenceEmbedderApplier.validate_inference_state(state)
    if state and "model_snapshot" not in state:
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved schema and asset identity without loading or running the encoder.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved schema and asset identity without loading or running the encoder."""
    if type(raw) is not dict:
        raise ValueError("Local fitted state must be a dictionary.")
    fields = {
        "type",
        "columns",
        "model_name",
        "embedding_dim",
        "normalize",
        "output_columns",
        "drop_original",
    }
    assets = {"model_snapshot", "model_sha256", "model_requirements"}
    fields |= assets if assets.intersection(raw) else set()
    if not local_state_fields(raw, "sentence_embedder", fields, allow_empty=True):
        return raw
    local_scalar(raw["normalize"], "normalize")
    local_scalar(raw["drop_original"], "drop_original")
    _validate_text_layout({**raw, "drop_original": bool(raw["drop_original"])})
    if not isinstance(raw["model_name"], str) or not raw["model_name"]:
        raise ValueError("Fitted sentence model name must be nonempty text.")
    width = raw["embedding_dim"]
    if type(width) is not int or width <= 0 or len(raw["output_columns"]) != width:
        raise ValueError("Fitted sentence embedding width disagrees with its columns.")
    if "model_snapshot" in raw:
        _validate_snapshot(raw)
    return raw

SentenceEmbedderCalculator

Bases: BaseCalculator

Resolve text columns and capture the pretrained encoder used by subsequent applies.

learns_from_data=False — the weights come pretrained and are never adjusted here — but fitting is not free: it performs the lazy sentence-transformers import and, on a cold cache, downloads the model. Native PyTorch serialization captures its weights, tokenizer and settings in memory, with a SHA256 identity and exact optional dependency versions. The existing local pipeline/model-set size budgets still apply to these bytes.

Schema inference inherits the base None result because the embedding width is only known after loading the model. Previewing the schema therefore does not load model weights and leaves the output unknown until runtime.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

@NodeRegistry.register("sentence_embedder", SentenceEmbedderApplier)
@node_meta(
    id="sentence_embedder",
    name="Sentence Embedder",
    category="Text",
    description=(
        "Encode text columns into dense semantic embeddings using a "
        "sentence-transformers model (default all-MiniLM-L6-v2). Requires the "
        "optional 'sentence-transformers' package."
    ),
    params={
        "columns": [],
        "model_name": "all-MiniLM-L6-v2",
        "normalize": True,
        "drop_original": False,
    },
    tags=["text", "nlp", "embeddings", "transformers"],
    learns_from_data=False,
)
class SentenceEmbedderCalculator(BaseCalculator):
    """Resolve text columns and capture the pretrained encoder used by subsequent applies.

    ``learns_from_data=False`` — the weights come pretrained and are never
    adjusted here — but fitting is not free: it performs the lazy
    ``sentence-transformers`` import and, on a cold cache, downloads the model.
    Native PyTorch serialization captures its weights, tokenizer and settings in
    memory, with a SHA256 identity and exact optional dependency versions. The
    existing local pipeline/model-set size budgets still apply to these bytes.

    Schema inference inherits the base ``None`` result because the embedding
    width is only known after loading the model. Previewing the schema therefore
    does not load model weights and leaves the output unknown until runtime.
    """

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> SentenceEmbedderArtifact:  # pylint: disable=arguments-differ
        """Resolve the configured text column names and record the model's embedding width.

        Uses the column-names-only resolver rather than the data-narrowing
        sibling, so a polars input is never converted to pandas just to fit. An
        empty or wholly-absent selection yields an empty artifact so the applier
        no-ops — and, importantly, skips loading the model at all.
        """
        valid_cols = resolve_fit_text_valid_columns(X, config, _y)
        if valid_cols is None:
            return {}

        artifact = _build_sentence_embedder_artifact(config, valid_cols)
        validate_text_output_names(X, artifact, valid_cols)
        return artifact

fit(X, _y, config)

Resolve the configured text column names and record the model's embedding width.

Uses the column-names-only resolver rather than the data-narrowing sibling, so a polars input is never converted to pandas just to fit. An empty or wholly-absent selection yields an empty artifact so the applier no-ops — and, importantly, skips loading the model at all.

Source code in skyulf-core/skyulf/preprocessing/vectorization/sentence_embedder.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> SentenceEmbedderArtifact:  # pylint: disable=arguments-differ
    """Resolve the configured text column names and record the model's embedding width.

    Uses the column-names-only resolver rather than the data-narrowing
    sibling, so a polars input is never converted to pandas just to fit. An
    empty or wholly-absent selection yields an empty artifact so the applier
    no-ops — and, importantly, skips loading the model at all.
    """
    valid_cols = resolve_fit_text_valid_columns(X, config, _y)
    if valid_cols is None:
        return {}

    artifact = _build_sentence_embedder_artifact(config, valid_cols)
    validate_text_output_names(X, artifact, valid_cols)
    return artifact

SimpleImputerApplier

Bases: BaseApplier

Apply fitted Simple Imputer fill values to missing values in selected columns.

The calculator artifact records per-column values for mean, median, most_frequent (also accepted as mode), or constant strategies. Missing columns seen during fitting are restored with their stored value, preserving the row count even when no input columns remain. Empty saved artifacts declare identity replay only when the recipe explicitly selects no columns; automatic or missing selections remain unreviewed. Spark supports only mean and constant artifacts and applies them with native expressions. All-missing means leave existing columns untouched.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

class SimpleImputerApplier(BaseApplier):
    """Apply fitted Simple Imputer fill values to missing values in selected columns.

    The calculator artifact records per-column values for ``mean``, ``median``,
    ``most_frequent`` (also accepted as ``mode``), or ``constant`` strategies.
    Missing columns seen during fitting are restored with their stored value,
    preserving the row count even when no input columns remain.
    Empty saved artifacts declare identity replay only when the recipe explicitly
    selects no columns; automatic or missing selections remain unreviewed.
    Spark supports only ``mean`` and ``constant`` artifacts and applies them
    with native expressions. All-missing means leave existing columns untouched.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return (
            _local_statistic_state(raw)
            if raw.get("strategy") in ("median", "most_frequent")
            else validate_state("SimpleImputer", raw)
        )

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        if not state:
            if not user_picked_no_columns(raw) or raw.get("_auto_columns"):
                raise ValueError("An empty imputer artifact requires explicit empty columns.")
            strategy = raw.get("strategy")
            if strategy in ("median", "most_frequent", "mode"):
                canonical = "median" if strategy == "median" else "most_frequent"
                return _imputation_config(
                    "SimpleImputer",
                    fitted_columns(raw, {"columns": []}),
                    {"strategy": canonical},
                )
        if state.get("strategy") in ("median", "most_frequent"):
            return _imputation_config("SimpleImputer", fitted_columns(raw, state), state)
        return portable_config("SimpleImputer", raw, state)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Fill nulls and NaNs with the fitted per-column values; ``y`` passes through."""
        return apply_dual_engine(
            X,
            params,
            {
                "polars": self._apply_polars,
                "pandas": self._apply_pandas,
                "spark": apply_spark_imputer,
            },
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        fill_values = params.get("fill_values", {})
        if not cols:
            return X, _y

        exprs: list[Any] = []
        for col in X.columns:
            if col in cols and col in fill_values:
                val = fill_values[col]
                # An all-null fitted column has no computable mean/median, so
                # its stored fill value is None. Skip it (leave the column
                # untouched) to mirror the pandas branch's `if val is None`
                # guard; fill_null(None) would raise.
                if val is None:
                    exprs.append(pl.col(col))
                    continue
                expr = pl.col(col).fill_null(val)
                if X.schema[col].is_float():
                    expr = expr.fill_nan(val)
                exprs.append(expr.alias(col))
            else:
                exprs.append(pl.col(col))

        # Restore columns that were present at fit time but missing in input X.
        exprs.extend(
            pl.repeat(pl.lit(fill_values[col]), pl.len()).alias(col)
            for col in cols
            if col not in X.columns and col in fill_values
        )

        return X.select(exprs), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        fill_values = params.get("fill_values", {})
        if not cols:
            return X, _y

        X_out = X.copy()
        for col in cols:
            val = fill_values.get(col)
            if val is None:
                continue
            if col not in X_out.columns:
                X_out[col] = val
            else:
                X_out[col] = _fill_pandas_series(X_out[col], val, params.get("strategy"))
        return X_out, _y

apply(X, _y, params)

Fill nulls and NaNs with the fitted per-column values; y passes through.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Fill nulls and NaNs with the fitted per-column values; ``y`` passes through."""
    return apply_dual_engine(
        X,
        params,
        {
            "polars": self._apply_polars,
            "pandas": self._apply_pandas,
            "spark": apply_spark_imputer,
        },
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    if not state:
        if not user_picked_no_columns(raw) or raw.get("_auto_columns"):
            raise ValueError("An empty imputer artifact requires explicit empty columns.")
        strategy = raw.get("strategy")
        if strategy in ("median", "most_frequent", "mode"):
            canonical = "median" if strategy == "median" else "most_frequent"
            return _imputation_config(
                "SimpleImputer",
                fitted_columns(raw, {"columns": []}),
                {"strategy": canonical},
            )
    if state.get("strategy") in ("median", "most_frequent"):
        return _imputation_config("SimpleImputer", fitted_columns(raw, state), state)
    return portable_config("SimpleImputer", raw, state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return (
        _local_statistic_state(raw)
        if raw.get("strategy") in ("median", "most_frequent")
        else validate_state("SimpleImputer", raw)
    )

SimpleImputerCalculator

Bases: BaseCalculator

Fit per-column values for filling missing data with sklearn-compatible strategies.

Supported strategy values are mean, median, most_frequent, and constant; mode is normalized to most_frequent. Use columns to select columns and fill_value with constant. Mean and median auto-select numeric columns and reject explicitly selected non-numeric columns. The other strategies can operate on all selected columns. Spark supports only mean and constant through a native aggregate; string and boolean constants require explicit compatible fill values.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

@NodeRegistry.register(
    "SimpleImputer",
    SimpleImputerApplier,
    execution_capabilities=tuple(
        ExecutionCapability(
            "spark",
            operation,
            "native",
            "preserve",
            "global" if operation == "fit" else "row",
            codec_version=1,
            config_match=(("strategy", strategy),),
        )
        for operation in ("fit", "apply")
        for strategy in ("mean", "constant")
    )
    + tuple(
        ExecutionCapability(
            engine,
            "apply",
            execution_kind,
            "preserve",
            "row",
            codec_version=1,
            config_match=(("strategy", strategy),),
        )
        for engine, execution_kind in (("pandas", "python_batch"), ("polars", "local"))
        for strategy in ("mean", "constant")
    )
    + tuple(
        ExecutionCapability(
            engine,
            "apply",
            execution_kind,
            "preserve",
            "row",
            config_match=(("strategy", "most_frequent"),),
        )
        for engine, execution_kind in (("pandas", "python_batch"), ("polars", "local"))
    )
    + tuple(
        ExecutionCapability(
            engine,
            "apply",
            "local",
            "preserve",
            "row",
            config_match=(("strategy", "median"),),
        )
        for engine in ("pandas", "polars")
    ),
)
@node_meta(
    id="SimpleImputer",
    name="Simple Imputer",
    category="Preprocessing",
    description="Imputes missing values using mean, median, or constant.",
    params={"strategy": "mean", "fill_value": None, "columns": []},
    learns_from_data=True,
)
class SimpleImputerCalculator(BaseCalculator):
    """Fit per-column values for filling missing data with sklearn-compatible strategies.

    Supported ``strategy`` values are ``mean``, ``median``, ``most_frequent``,
    and ``constant``; ``mode`` is normalized to ``most_frequent``. Use
    ``columns`` to select columns and ``fill_value`` with ``constant``.
    Mean and median auto-select numeric columns and reject explicitly selected
    non-numeric columns. The other strategies can operate on all selected columns.
    Spark supports only ``mean`` and ``constant`` through a native aggregate;
    string and boolean constants require explicit compatible fill values.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` and promote float-only imputation targets to ``float64``."""
        strategy = config.get("strategy", "mean")
        if strategy == "mode":
            strategy = "most_frequent"
        if strategy not in {"mean", "median"}:
            return input_schema
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> SimpleImputerArtifact:  # pylint: disable=arguments-differ
        """Resolve strategy and columns, then fit fill values on the active engine.

        ``mode`` is accepted as an alias of ``most_frequent``; mean/median are
        restricted to numeric columns.
        """
        return cast(
            SimpleImputerArtifact,
            fit_dual_engine(
                X,
                config,
                {
                    "polars": self._fit_polars_input,
                    "pandas": self._fit_pandas_input,
                    "spark": fit_spark_imputer,
                },
            ),
        )

    @staticmethod
    def _prepare_local(X: Any, config: dict[str, Any]) -> dict[str, Any] | None:
        """Keep existing local column/strategy resolution separate from Spark preparation."""
        if user_picked_no_columns(config):
            return None

        strategy = config.get("strategy", "mean")
        if strategy == "mode":
            strategy = "most_frequent"
        fill_value = config.get("fill_value")

        cols = _resolve_simple_columns(X, config, strategy)
        if not cols:
            return None

        if strategy in {"mean", "median"}:
            numeric = set(auto_detect_numeric_columns(X))
            invalid = [col for col in cols if col not in numeric]
            if invalid:
                raise ValueError(
                    f"SimpleImputer strategy '{strategy}' requires numeric columns; "
                    f"non-numeric columns selected: {invalid}."
                )

        # Stash resolved-once values into params so dispatched fits don't redo work.
        merged = dict(config)
        merged["_resolved_strategy"] = strategy
        merged["_resolved_cols"] = cols
        merged["_resolved_fill_value"] = fill_value

        return merged

    @staticmethod
    def _fit_polars_input(X: Any, y: Any, config: dict[str, Any]) -> dict[str, Any]:
        """Prepare the existing Polars contract before its fit implementation."""
        prepared = SimpleImputerCalculator._prepare_local(X, config)
        return {} if prepared is None else SimpleImputerCalculator._fit_polars(X, y, prepared)

    @staticmethod
    def _fit_pandas_input(X: Any, y: Any, config: dict[str, Any]) -> dict[str, Any]:
        """Prepare the existing pandas contract before its fit implementation."""
        prepared = SimpleImputerCalculator._prepare_local(X, config)
        return {} if prepared is None else SimpleImputerCalculator._fit_pandas(X, y, prepared)

    @staticmethod
    def _fit_polars(X: Any, _y: Any, params: dict[str, Any]) -> dict[str, Any]:
        cols: list[str] = params["_resolved_cols"]
        strategy: str = params["_resolved_strategy"]
        fill_value = params["_resolved_fill_value"]

        fill_values = _compute_polars_fill_values(X, cols, strategy, fill_value)
        missing_counts, total_missing = _polars_missing_counts(X, cols)

        return {
            "type": "simple_imputer",
            "strategy": strategy,
            "fill_values": fill_values,
            "columns": cols,
            "missing_counts": missing_counts,
            "total_missing": total_missing,
        }

    @staticmethod
    def _fit_pandas(X: Any, _y: Any, params: dict[str, Any]) -> dict[str, Any]:
        cols: list[str] = params["_resolved_cols"]
        strategy: str = params["_resolved_strategy"]
        fill_value = params["_resolved_fill_value"]

        # Mean/median: extra safety filter to numeric columns only.
        if strategy in ("mean", "median"):
            # Automatic selection already applied its cardinality heuristics.
            # Do not discard explicitly selected binary or constant columns here.
            numeric = set(detect_numeric_columns(X, exclude_binary=False, exclude_constant=False))
            cols = [c for c in cols if c in numeric]
            if not cols:
                return {}

        imputer = SimpleImputer(
            strategy=strategy,
            fill_value=fill_value,
            keep_empty_features=strategy == "constant",
        )
        fit_data = (
            decimal_columns_to_float(X[cols], cols) if strategy in {"mean", "median"} else X[cols]
        )
        if strategy == "most_frequent":
            # Preserve nullable integers/booleans as scalar values and normalize
            # pd.NA, which sklearn's object-array missing mask cannot compare.
            fit_data = X[cols].to_numpy(dtype=object, na_value=float("nan"))
        imputer.fit(fit_data)

        if strategy == "constant" and fill_value is not None:
            # Validation still runs, but heldout missingness must not choose a constant.
            fill_values = dict.fromkeys(cols, fill_value)
        else:
            # Keeping empty constant columns preserves sklearn's dtype-based defaults.
            statistics = imputer.statistics_.tolist()
            fill_values = dict(zip(cols, statistics, strict=True))
        missing_counts = X[cols].isnull().sum().to_dict()
        total_missing = int(sum(missing_counts.values()))

        return {
            "type": "simple_imputer",
            "strategy": strategy,
            "fill_values": fill_values,
            "columns": cols,
            "missing_counts": missing_counts,
            "total_missing": total_missing,
        }

fit(X, _y, config)

Resolve strategy and columns, then fit fill values on the active engine.

mode is accepted as an alias of most_frequent; mean/median are restricted to numeric columns.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> SimpleImputerArtifact:  # pylint: disable=arguments-differ
    """Resolve strategy and columns, then fit fill values on the active engine.

    ``mode`` is accepted as an alias of ``most_frequent``; mean/median are
    restricted to numeric columns.
    """
    return cast(
        SimpleImputerArtifact,
        fit_dual_engine(
            X,
            config,
            {
                "polars": self._fit_polars_input,
                "pandas": self._fit_pandas_input,
                "spark": fit_spark_imputer,
            },
        ),
    )

infer_output_schema(input_schema, config)

Return input_schema and promote float-only imputation targets to float64.

Source code in skyulf-core/skyulf/preprocessing/imputation/simple.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` and promote float-only imputation targets to ``float64``."""
    strategy = config.get("strategy", "mean")
    if strategy == "mode":
        strategy = "most_frequent"
    if strategy not in {"mean", "median"}:
        return input_schema
    return promote_configured_columns_to_float64(input_schema, config)

SimpleTransformationApplier

Bases: BaseApplier

Apply the configured per-column math transformations (log, sqrt, ...).

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

class SimpleTransformationApplier(BaseApplier):
    """Apply the configured per-column math transformations (log, sqrt, ...)."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect ordered native rules without consuming iterators or normalizing state."""
        local_state_fields(raw, "simple_transformation", {"type", "transformations"})
        transformations = raw["transformations"]
        if transformations is None:
            return raw
        if type(transformations) not in (list, tuple):
            raise ValueError("SimpleTransformation rules must be a list or tuple.")
        for item in transformations:
            _validate_simple_rule(item)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe native pointwise math as local row-preserving inference."""
        if engine not in ("pandas", "polars"):
            return None
        SimpleTransformationApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Run each configured transformation on the active engine; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        transformations = params.get("transformations", [])
        if not transformations:
            return X, _y

        X_out = X
        for item in transformations:
            col = item.get("column")
            method = item.get("method")
            if col not in X_out.columns:
                continue
            op = _POLARS_OPS.get(method)
            if op is None:
                continue
            X_out = _apply_polars_op(X_out, item, op)
        return X_out, _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        transformations = params.get("transformations", [])
        if not transformations:
            return X, _y

        df_out = X.copy()
        for item in transformations:
            col = item.get("column")
            method = item.get("method")
            if col not in df_out.columns:
                continue
            op = _PANDAS_OPS.get(method)
            if op is None:
                continue
            df_out[col] = op(pd.to_numeric(df_out[col], errors="coerce"), item)
        return df_out, _y

apply(X, _y, params)

Run each configured transformation on the active engine; y passes through.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Run each configured transformation on the active engine; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe native pointwise math as local row-preserving inference.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe native pointwise math as local row-preserving inference."""
    if engine not in ("pandas", "polars"):
        return None
    SimpleTransformationApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect ordered native rules without consuming iterators or normalizing state.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect ordered native rules without consuming iterators or normalizing state."""
    local_state_fields(raw, "simple_transformation", {"type", "transformations"})
    transformations = raw["transformations"]
    if transformations is None:
        return raw
    if type(transformations) not in (list, tuple):
        raise ValueError("SimpleTransformation rules must be a list or tuple.")
    for item in transformations:
        _validate_simple_rule(item)
    return raw

SimpleTransformationCalculator

Bases: BaseCalculator

Package the configured transformations into an artifact without reading the data.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

@NodeRegistry.register("SimpleTransformation", SimpleTransformationApplier)
@node_meta(
    id="SimpleTransformation",
    name="Simple Transformation",
    category="Preprocessing",
    description="Apply simple mathematical transformations (log, sqrt, etc.).",
    params={"func": "log", "columns": []},
    learns_from_data=False,
)
class SimpleTransformationCalculator(BaseCalculator):
    """Package the configured transformations into an artifact without reading the data."""

    _FLOAT_METHODS = {
        "square",
        "log",
        "sqrt",
        "square_root",
        "cube_root",
        "reciprocal",
        "exp",
        "exponential",
    }

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return updated dtypes for transformed numeric columns."""
        schema = input_schema
        touched: set[str] = set()
        for item in config.get("transformations", []):
            col = item.get("column")
            method = item.get("method")
            if col not in schema.columns or method is None:
                continue
            if method in self._FLOAT_METHODS:
                touched.add(col)

        for col in touched:
            schema = schema.with_dtype(col, "float64")
        return schema

    def fit(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        config: dict[str, Any],
    ) -> SimpleTransformationArtifact:
        """Return the ``transformations`` config as-is; nothing is fitted."""
        # Config: {'transformations': [{'column': 'col1', 'method': 'log'}, ...]}
        return {
            "type": "simple_transformation",
            "transformations": config.get("transformations", []),
        }

fit(df, config)

Return the transformations config as-is; nothing is fitted.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

def fit(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    config: dict[str, Any],
) -> SimpleTransformationArtifact:
    """Return the ``transformations`` config as-is; nothing is fitted."""
    # Config: {'transformations': [{'column': 'col1', 'method': 'log'}, ...]}
    return {
        "type": "simple_transformation",
        "transformations": config.get("transformations", []),
    }

infer_output_schema(input_schema, config)

Return updated dtypes for transformed numeric columns.

Source code in skyulf-core/skyulf/preprocessing/transformations/simple.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return updated dtypes for transformed numeric columns."""
    schema = input_schema
    touched: set[str] = set()
    for item in config.get("transformations", []):
        col = item.get("column")
        method = item.get("method")
        if col not in schema.columns or method is None:
            continue
        if method in self._FLOAT_METHODS:
            touched.add(col)

    for col in touched:
        schema = schema.with_dtype(col, "float64")
    return schema

SkyulfSchema dataclass

Immutable schema description.

Attributes:

Name	Type	Description
columns	tuple[str, ...]	Ordered list of column names.
dtypes	dict[str, str]	Mapping of column name → string dtype label (engine-agnostic; e.g. "int64", "float64", "string", "category", "datetime", "bool", or "unknown"). A column may be present in columns but absent from dtypes when its type is unknown.

Source code in skyulf-core/skyulf/core/schema.py

@dataclass(frozen=True)
class SkyulfSchema:
    """Immutable schema description.

    Attributes:
        columns: Ordered list of column names.
        dtypes: Mapping of column name → string dtype label
            (engine-agnostic; e.g. ``"int64"``, ``"float64"``, ``"string"``,
            ``"category"``, ``"datetime"``, ``"bool"``, or ``"unknown"``).
            A column may be present in ``columns`` but absent from
            ``dtypes`` when its type is unknown.
    """

    columns: tuple[str, ...]
    dtypes: dict[str, str] = field(default_factory=dict)

    # ---- Constructors -----------------------------------------------------

    @classmethod
    def from_columns(
        cls, columns: Iterable[str], dtypes: dict[str, str] | None = None
    ) -> "SkyulfSchema":
        """Build a schema from an iterable of ``columns`` and an optional ``dtypes`` mapping."""
        cols = tuple(columns)
        return cls(columns=cols, dtypes=dict(dtypes or {}))

    @classmethod
    def from_dataframe(cls, df: Any) -> "SkyulfSchema":
        """Extract schema metadata from local or Spark frames without collecting rows."""
        if is_spark_input(df):
            native = df.to_native() if isinstance(df, SkyulfSparkWrapper) else df
            labels = {
                "byte": "int8",
                "short": "int16",
                "integer": "int32",
                "long": "int64",
                "float": "float32",
                "double": "float64",
                "boolean": "bool",
                "timestamp": "datetime",
                "timestamp_ntz": "datetime",
            }
            return cls(
                tuple(native.columns),
                {
                    field.name: labels.get(field.dataType.typeName(), field.dataType.simpleString())
                    for field in native.schema.fields
                },
            )
        raw_cols = getattr(df, "columns", None)
        cols = list(raw_cols) if raw_cols is not None else []
        dtypes = _extract_pandas_dtypes(df)
        if not dtypes:
            dtypes = _extract_polars_dtypes(df)
        return cls(columns=tuple(cols), dtypes=dtypes)

    # ---- Mutations (return new instances) ---------------------------------

    def drop(self, names: Iterable[str]) -> "SkyulfSchema":
        """Return a new schema with the ``names`` columns (and their dtypes) removed."""
        drop_set = set(names)
        new_cols = tuple(c for c in self.columns if c not in drop_set)
        new_dtypes = {k: v for k, v in self.dtypes.items() if k not in drop_set}
        return replace(self, columns=new_cols, dtypes=new_dtypes)

    def add(self, name: str, dtype: str = "unknown") -> "SkyulfSchema":
        """Return a new schema with ``name`` added under ``dtype``; unchanged if already present."""
        if name in self.columns:
            return self
        new_dtypes = dict(self.dtypes)
        new_dtypes[name] = dtype
        return replace(self, columns=self.columns + (name,), dtypes=new_dtypes)

    def rename(self, mapping: dict[str, str]) -> "SkyulfSchema":
        """Return a new schema with columns renamed according to ``mapping``.

        Raises:
            ValueError: If the rename would produce duplicate column names.
        """
        new_cols = tuple(mapping.get(c, c) for c in self.columns)
        if len(set(new_cols)) != len(new_cols):
            seen: set[str] = set()
            collisions: set[str] = set()
            for c in new_cols:
                if c in seen:
                    collisions.add(c)
                seen.add(c)
            raise ValueError(
                "Schema rename() would produce duplicate column name(s): "
                f"{sorted(collisions)}. Rename mapping: {mapping}"
            )
        new_dtypes: dict[str, str] = {}
        for k, v in self.dtypes.items():
            new_dtypes[mapping.get(k, k)] = v
        return replace(self, columns=new_cols, dtypes=new_dtypes)

    def with_dtype(self, name: str, dtype: str) -> "SkyulfSchema":
        """Return a new schema with ``name``'s dtype set to ``dtype``; unchanged if absent."""
        if name not in self.columns:
            return self
        new_dtypes = dict(self.dtypes)
        new_dtypes[name] = dtype
        return replace(self, dtypes=new_dtypes)

    # ---- Queries ----------------------------------------------------------

    def has(self, name: str) -> bool:
        """Return ``True`` if ``name`` is a column in this schema."""
        return name in self.columns

    def column_list(self) -> list[str]:
        """Return the column names as a plain ``list``."""
        return list(self.columns)

    def __contains__(self, item: object) -> bool:
        """Support ``in``: return ``True`` if ``item`` is a column name in this schema."""
        return item in self.columns

    def __len__(self) -> int:
        """Return the number of columns in this schema."""
        return len(self.columns)

    # ---- Contract validation ---------------------------------------------

    def assert_compatible(
        self,
        actual: "SkyulfSchema",
        *,
        check_dtypes: bool = False,
        check_order: bool = False,
        where: str = "input",
    ) -> None:
        """Validate that ``actual`` satisfies this (expected) schema.

        ``self`` is the expected schema (e.g. what an Applier was fitted on);
        ``actual`` is the schema observed at apply time. Raises
        :class:`SchemaMismatchError` describing every discrepancy. Presence of
        the expected columns is always checked; dtype and column-order checks
        are opt-in to keep the default contract permissive and non-breaking.

        Args:
            actual: The schema observed at runtime.
            check_dtypes: Also compare dtype labels for shared columns.
            check_order: Also require shared columns in the same relative order.
            where: Label used in the error message (e.g. ``"input"``).
        """
        missing, unexpected = _presence_diff(self, actual)
        dtype_mismatches = _dtype_mismatches(self, actual) if check_dtypes else {}
        order_mismatch = _check_order(self, actual, check_order, missing)

        if missing or unexpected or dtype_mismatches or order_mismatch:
            raise SchemaMismatchError(
                _format_mismatch(where, missing, unexpected, dtype_mismatches, order_mismatch),
                missing=missing,
                unexpected=unexpected,
                dtype_mismatches=dtype_mismatches,
                order_mismatch=order_mismatch,
            )

__contains__(item)

Support in: return True if item is a column name in this schema.

Source code in skyulf-core/skyulf/core/schema.py

def __contains__(self, item: object) -> bool:
    """Support ``in``: return ``True`` if ``item`` is a column name in this schema."""
    return item in self.columns

__len__()

Return the number of columns in this schema.

Source code in skyulf-core/skyulf/core/schema.py

def __len__(self) -> int:
    """Return the number of columns in this schema."""
    return len(self.columns)

add(name, dtype='unknown')

Return a new schema with name added under dtype; unchanged if already present.

Source code in skyulf-core/skyulf/core/schema.py

def add(self, name: str, dtype: str = "unknown") -> "SkyulfSchema":
    """Return a new schema with ``name`` added under ``dtype``; unchanged if already present."""
    if name in self.columns:
        return self
    new_dtypes = dict(self.dtypes)
    new_dtypes[name] = dtype
    return replace(self, columns=self.columns + (name,), dtypes=new_dtypes)

assert_compatible(actual, *, check_dtypes=False, check_order=False, where='input')

Validate that actual satisfies this (expected) schema.

self is the expected schema (e.g. what an Applier was fitted on); actual is the schema observed at apply time. Raises :class:SchemaMismatchError describing every discrepancy. Presence of the expected columns is always checked; dtype and column-order checks are opt-in to keep the default contract permissive and non-breaking.

Parameters:

Name	Type	Description	Default
actual	SkyulfSchema	The schema observed at runtime.	required
check_dtypes	bool	Also compare dtype labels for shared columns.	False
check_order	bool	Also require shared columns in the same relative order.	False
where	str	Label used in the error message (e.g. "input").	'input'

Source code in skyulf-core/skyulf/core/schema.py

def assert_compatible(
    self,
    actual: "SkyulfSchema",
    *,
    check_dtypes: bool = False,
    check_order: bool = False,
    where: str = "input",
) -> None:
    """Validate that ``actual`` satisfies this (expected) schema.

    ``self`` is the expected schema (e.g. what an Applier was fitted on);
    ``actual`` is the schema observed at apply time. Raises
    :class:`SchemaMismatchError` describing every discrepancy. Presence of
    the expected columns is always checked; dtype and column-order checks
    are opt-in to keep the default contract permissive and non-breaking.

    Args:
        actual: The schema observed at runtime.
        check_dtypes: Also compare dtype labels for shared columns.
        check_order: Also require shared columns in the same relative order.
        where: Label used in the error message (e.g. ``"input"``).
    """
    missing, unexpected = _presence_diff(self, actual)
    dtype_mismatches = _dtype_mismatches(self, actual) if check_dtypes else {}
    order_mismatch = _check_order(self, actual, check_order, missing)

    if missing or unexpected or dtype_mismatches or order_mismatch:
        raise SchemaMismatchError(
            _format_mismatch(where, missing, unexpected, dtype_mismatches, order_mismatch),
            missing=missing,
            unexpected=unexpected,
            dtype_mismatches=dtype_mismatches,
            order_mismatch=order_mismatch,
        )

column_list()

Return the column names as a plain list.

Source code in skyulf-core/skyulf/core/schema.py

def column_list(self) -> list[str]:
    """Return the column names as a plain ``list``."""
    return list(self.columns)

drop(names)

Return a new schema with the names columns (and their dtypes) removed.

Source code in skyulf-core/skyulf/core/schema.py

def drop(self, names: Iterable[str]) -> "SkyulfSchema":
    """Return a new schema with the ``names`` columns (and their dtypes) removed."""
    drop_set = set(names)
    new_cols = tuple(c for c in self.columns if c not in drop_set)
    new_dtypes = {k: v for k, v in self.dtypes.items() if k not in drop_set}
    return replace(self, columns=new_cols, dtypes=new_dtypes)

from_columns(columns, dtypes=None) classmethod

Build a schema from an iterable of columns and an optional dtypes mapping.

Source code in skyulf-core/skyulf/core/schema.py

@classmethod
def from_columns(
    cls, columns: Iterable[str], dtypes: dict[str, str] | None = None
) -> "SkyulfSchema":
    """Build a schema from an iterable of ``columns`` and an optional ``dtypes`` mapping."""
    cols = tuple(columns)
    return cls(columns=cols, dtypes=dict(dtypes or {}))

from_dataframe(df) classmethod

Extract schema metadata from local or Spark frames without collecting rows.

Source code in skyulf-core/skyulf/core/schema.py

@classmethod
def from_dataframe(cls, df: Any) -> "SkyulfSchema":
    """Extract schema metadata from local or Spark frames without collecting rows."""
    if is_spark_input(df):
        native = df.to_native() if isinstance(df, SkyulfSparkWrapper) else df
        labels = {
            "byte": "int8",
            "short": "int16",
            "integer": "int32",
            "long": "int64",
            "float": "float32",
            "double": "float64",
            "boolean": "bool",
            "timestamp": "datetime",
            "timestamp_ntz": "datetime",
        }
        return cls(
            tuple(native.columns),
            {
                field.name: labels.get(field.dataType.typeName(), field.dataType.simpleString())
                for field in native.schema.fields
            },
        )
    raw_cols = getattr(df, "columns", None)
    cols = list(raw_cols) if raw_cols is not None else []
    dtypes = _extract_pandas_dtypes(df)
    if not dtypes:
        dtypes = _extract_polars_dtypes(df)
    return cls(columns=tuple(cols), dtypes=dtypes)

has(name)

Return True if name is a column in this schema.

Source code in skyulf-core/skyulf/core/schema.py

def has(self, name: str) -> bool:
    """Return ``True`` if ``name`` is a column in this schema."""
    return name in self.columns

rename(mapping)

Return a new schema with columns renamed according to mapping.

Raises:

Type	Description
ValueError	If the rename would produce duplicate column names.

Source code in skyulf-core/skyulf/core/schema.py

def rename(self, mapping: dict[str, str]) -> "SkyulfSchema":
    """Return a new schema with columns renamed according to ``mapping``.

    Raises:
        ValueError: If the rename would produce duplicate column names.
    """
    new_cols = tuple(mapping.get(c, c) for c in self.columns)
    if len(set(new_cols)) != len(new_cols):
        seen: set[str] = set()
        collisions: set[str] = set()
        for c in new_cols:
            if c in seen:
                collisions.add(c)
            seen.add(c)
        raise ValueError(
            "Schema rename() would produce duplicate column name(s): "
            f"{sorted(collisions)}. Rename mapping: {mapping}"
        )
    new_dtypes: dict[str, str] = {}
    for k, v in self.dtypes.items():
        new_dtypes[mapping.get(k, k)] = v
    return replace(self, columns=new_cols, dtypes=new_dtypes)

with_dtype(name, dtype)

Return a new schema with name's dtype set to dtype; unchanged if absent.

Source code in skyulf-core/skyulf/core/schema.py

def with_dtype(self, name: str, dtype: str) -> "SkyulfSchema":
    """Return a new schema with ``name``'s dtype set to ``dtype``; unchanged if absent."""
    if name not in self.columns:
        return self
    new_dtypes = dict(self.dtypes)
    new_dtypes[name] = dtype
    return replace(self, dtypes=new_dtypes)

SplitApplier

Bases: BaseApplier

Route the input to the right :class:DataSplitter entry point.

Source code in skyulf-core/skyulf/preprocessing/split.py

class SplitApplier(BaseApplier):
    """Route the input to the right :class:`DataSplitter` entry point."""

    def apply(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        params: dict[str, Any],
        *,
        sample_weight: Any = None,
    ) -> SplitDataset:
        """Split ``df`` into train/test and optional validation.

        Routes three ways: an ``(X, y)`` tuple is split as a pair; a plain frame
        carrying a configured ``target_column`` has that column lifted into ``y``
        first; anything else is split row-wise with the target left in place.
        """
        target_col = params.get("target_column")
        splitter = _build_splitter(params)

        if isinstance(df, tuple) and len(df) == 2:
            X, y = df
            return splitter.split_xy(cast(Any, X), y, sample_weight=sample_weight)

        # If a target column is configured and present, split features from target
        # so downstream nodes see real (X, y) tuples — not the placeholder
        # (df, None) shape that gets silently collapsed by transformers.
        if target_col and hasattr(df, "columns") and target_col in list(cast(Any, df).columns):
            frame = cast(Any, df)
            # `.drop(columns=[...])` is pandas-only syntax; polars' `.drop()`
            # takes column names positionally and has no `columns=` kwarg.
            X = frame.drop(target_col) if is_polars(frame) else frame.drop(columns=[target_col])
            y = frame[target_col]
            return splitter.split_xy(X, y, sample_weight=sample_weight)

        return splitter.split(cast(Any, df), sample_weight=sample_weight)

apply(df, params, *, sample_weight=None)

Split df into train/test and optional validation.

Routes three ways: an (X, y) tuple is split as a pair; a plain frame carrying a configured target_column has that column lifted into y first; anything else is split row-wise with the target left in place.

Source code in skyulf-core/skyulf/preprocessing/split.py

def apply(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    params: dict[str, Any],
    *,
    sample_weight: Any = None,
) -> SplitDataset:
    """Split ``df`` into train/test and optional validation.

    Routes three ways: an ``(X, y)`` tuple is split as a pair; a plain frame
    carrying a configured ``target_column`` has that column lifted into ``y``
    first; anything else is split row-wise with the target left in place.
    """
    target_col = params.get("target_column")
    splitter = _build_splitter(params)

    if isinstance(df, tuple) and len(df) == 2:
        X, y = df
        return splitter.split_xy(cast(Any, X), y, sample_weight=sample_weight)

    # If a target column is configured and present, split features from target
    # so downstream nodes see real (X, y) tuples — not the placeholder
    # (df, None) shape that gets silently collapsed by transformers.
    if target_col and hasattr(df, "columns") and target_col in list(cast(Any, df).columns):
        frame = cast(Any, df)
        # `.drop(columns=[...])` is pandas-only syntax; polars' `.drop()`
        # takes column names positionally and has no `columns=` kwarg.
        X = frame.drop(target_col) if is_polars(frame) else frame.drop(columns=[target_col])
        y = frame[target_col]
        return splitter.split_xy(X, y, sample_weight=sample_weight)

    return splitter.split(cast(Any, df), sample_weight=sample_weight)

SplitCalculator

Bases: BaseCalculator

Carry the split configuration into the artifact; splitting happens at apply time.

Source code in skyulf-core/skyulf/preprocessing/split.py

@NodeRegistry.register("Split", SplitApplier)
@NodeRegistry.register("TrainTestSplitter", SplitApplier)
@node_meta(
    id="TrainTestSplitter",
    name="Train/Test Split",
    category="Data Operations",
    description="Split the dataset into training and testing sets.",
    params={
        "test_size": 0.2,
        "validation_size": 0.0,
        "random_state": DEFAULT_RANDOM_STATE,
        "shuffle": True,
        "stratify": False,
        "target_column": "target",
    },
    learns_from_data=False,
    is_splitter=True,
)
class SplitCalculator(BaseCalculator):
    """Carry the split configuration into the artifact; splitting happens at apply time."""

    def fit(
        self, df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any, config: dict[str, Any]
    ) -> SplitArtifact:
        """Return a typed artifact and warn about ignored public configuration keys.

        ``type`` and underscore-prefixed routing metadata are ignored quietly.
        Use ``stratify=True`` with ``target_column`` to request stratification;
        ``stratify_col`` belongs to ``DataSplitter``, not this node's config.
        """
        # No learning from data; pass through known split params from config.
        # Constructed explicitly so the artifact shape matches SplitArtifact
        # rather than echoing arbitrary user keys back into the params dict.
        unknown = sorted(
            key
            for key in config
            if key not in _SPLIT_CONFIG_KEYS and key != "type" and not key.startswith("_")
        )
        if unknown:
            logger.warning(
                "TrainTestSplitter ignored unrecognized config keys: %s. Supported keys: %s.",
                unknown,
                ", ".join(_SPLIT_CONFIG_KEYS),
            )
        artifact: SplitArtifact = {"type": "split"}
        for key in _SPLIT_CONFIG_KEYS:
            if key in config:
                artifact[key] = config[key]  # type: ignore[literal-required]
        return artifact

    def infer_output_schema(
        self,
        input_schema: SkyulfSchema,
        config: dict[str, Any],
    ) -> SkyulfSchema | None:
        """Return ``input_schema``: every split frame keeps the full column set."""
        # Split produces a SplitDataset whose train/test/val frames carry
        # the full input schema. Downstream consumers see the same columns.
        return input_schema

fit(df, config)

Return a typed artifact and warn about ignored public configuration keys.

type and underscore-prefixed routing metadata are ignored quietly. Use stratify=True with target_column to request stratification; stratify_col belongs to DataSplitter, not this node's config.

Source code in skyulf-core/skyulf/preprocessing/split.py

def fit(
    self, df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any, config: dict[str, Any]
) -> SplitArtifact:
    """Return a typed artifact and warn about ignored public configuration keys.

    ``type`` and underscore-prefixed routing metadata are ignored quietly.
    Use ``stratify=True`` with ``target_column`` to request stratification;
    ``stratify_col`` belongs to ``DataSplitter``, not this node's config.
    """
    # No learning from data; pass through known split params from config.
    # Constructed explicitly so the artifact shape matches SplitArtifact
    # rather than echoing arbitrary user keys back into the params dict.
    unknown = sorted(
        key
        for key in config
        if key not in _SPLIT_CONFIG_KEYS and key != "type" and not key.startswith("_")
    )
    if unknown:
        logger.warning(
            "TrainTestSplitter ignored unrecognized config keys: %s. Supported keys: %s.",
            unknown,
            ", ".join(_SPLIT_CONFIG_KEYS),
        )
    artifact: SplitArtifact = {"type": "split"}
    for key in _SPLIT_CONFIG_KEYS:
        if key in config:
            artifact[key] = config[key]  # type: ignore[literal-required]
    return artifact

infer_output_schema(input_schema, config)

Return input_schema: every split frame keeps the full column set.

Source code in skyulf-core/skyulf/preprocessing/split.py

def infer_output_schema(
    self,
    input_schema: SkyulfSchema,
    config: dict[str, Any],
) -> SkyulfSchema | None:
    """Return ``input_schema``: every split frame keeps the full column set."""
    # Split produces a SplitDataset whose train/test/val frames carry
    # the full input schema. Downstream consumers see the same columns.
    return input_schema

StandardScalerApplier

Bases: BaseApplier

Standardize the selected columns using the fitted mean and scale.

Honors with_mean/with_std; X passes through unchanged when the enabled flags have no fitted statistics, and y is never modified. Spark validates portable state and applies native expressions without a data action. Its selected numeric columns become double when scaling is enabled.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

class StandardScalerApplier(BaseApplier):
    """Standardize the selected columns using the fitted mean and scale.

    Honors ``with_mean``/``with_std``; ``X`` passes through unchanged when the
    enabled flags have no fitted statistics, and ``y`` is never modified.
    Spark validates portable state and applies native expressions without a
    data action. Its selected numeric columns become double when scaling is enabled.
    """

    @staticmethod
    def validate_fitted_state(raw: dict) -> dict:
        """Inspect this node's supported saved state without fitting or applying data."""
        return validate_state("StandardScaler", raw)

    @staticmethod
    def resolve_fitted_config(raw: dict, state: dict) -> dict:
        """Bind inference configuration to this node's inspected fitted artifact."""
        return portable_config("StandardScaler", raw, state)

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Center and scale ``X`` with the fitted statistics on the active engine."""
        return apply_dual_engine(
            X,
            params,
            {
                "polars": self._apply_polars,
                "pandas": self._apply_pandas,
                "spark": apply_spark_standard,
            },
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        mean = params.get("mean")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        with_mean = params.get("with_mean", True)
        with_std = params.get("with_std", True)
        if _needs_fitted_artifact(valid, with_mean, mean, with_std, scale):
            return X, _y

        mean_arr = np.array(mean) if mean is not None else np.zeros(len(cols))
        scale_arr = np.array(scale) if scale is not None else np.ones(len(cols))
        schema = X.collect_schema()

        def _standardized_expr(col_name: str) -> Any:
            idx = cols.index(col_name)
            e: Any = pl.col(col_name)
            if with_mean:
                e = e - mean_arr[idx]
            if with_std:
                s = scale_arr[idx]
                divided = e / (s if s != 0 else 1.0)
                dtype = schema[col_name]
                if s != 0 and (
                    dtype in (pl.Float32, pl.Float64) or (dtype.is_integer() and dtype != pl.Int128)
                ):
                    output_dtype = X.lazy().select(divided).collect_schema()[col_name]
                    if output_dtype in (pl.Float32, pl.Float64):
                        # Use Polars' native promotion with batch-independent NumPy division.
                        divided = cast(
                            pl.Expr,
                            np.divide(e, scale_arr[idx : idx + 1], dtype=str(output_dtype).lower()),
                        )
                e = divided
            return e.alias(col_name)

        exprs = [_standardized_expr(col_name) for col_name in valid]
        return X.with_columns(exprs), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        mean = params.get("mean")
        scale = params.get("scale")
        valid = resolve_valid_columns(X, cols)
        with_mean = params.get("with_mean", True)
        with_std = params.get("with_std", True)
        if not (with_mean or with_std):
            return X, _y
        if _needs_fitted_artifact(valid, with_mean, mean, with_std, scale):
            return X, _y

        X_out = X.copy()
        col_indices = [cols.index(c) for c in valid]
        subset = decimal_columns_to_float(X_out[valid], valid)
        vals = subset.to_numpy(dtype=np.float64, na_value=np.nan)
        if with_mean:
            vals = vals - np.array(mean)[col_indices]
        if with_std:
            vals = vals / safe_scale(np.array(scale)[col_indices].copy())

        X_out[valid] = vals
        return X_out, _y

apply(X, _y, params)

Center and scale X with the fitted statistics on the active engine.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Center and scale ``X`` with the fitted statistics on the active engine."""
    return apply_dual_engine(
        X,
        params,
        {
            "polars": self._apply_polars,
            "pandas": self._apply_pandas,
            "spark": apply_spark_standard,
        },
    )

resolve_fitted_config(raw, state) staticmethod

Bind inference configuration to this node's inspected fitted artifact.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

@staticmethod
def resolve_fitted_config(raw: dict, state: dict) -> dict:
    """Bind inference configuration to this node's inspected fitted artifact."""
    return portable_config("StandardScaler", raw, state)

validate_fitted_state(raw) staticmethod

Inspect this node's supported saved state without fitting or applying data.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

@staticmethod
def validate_fitted_state(raw: dict) -> dict:
    """Inspect this node's supported saved state without fitting or applying data."""
    return validate_state("StandardScaler", raw)

StandardScalerCalculator

Bases: BaseCalculator

Fit per-column statistics using sklearn locally or native Spark aggregates.

Spark uses population variance (ddof=0), ignores null/NaN observations, and returns at most two statistics rows to the driver. Both options default to True. Empty explicit training selections produce a no-op artifact; selected features with zero training rows cannot be fitted.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

@NodeRegistry.register(
    "StandardScaler",
    StandardScalerApplier,
    execution_capabilities=tuple(
        ExecutionCapability(
            "spark",
            operation,
            "native",
            "preserve",
            "global" if operation == "fit" else "row",
            codec_version=1,
            config_match=(("with_mean", with_mean), ("with_std", with_std)),
        )
        for operation in ("fit", "apply")
        for with_mean in (True, False)
        for with_std in (True, False)
    )
    + tuple(
        ExecutionCapability(
            engine,
            "apply",
            execution_kind,
            "preserve",
            "row",
            codec_version=1,
            config_match=(("with_mean", with_mean), ("with_std", with_std)),
        )
        for engine, execution_kind in (("pandas", "python_batch"), ("polars", "local"))
        for with_mean in (True, False)
        for with_std in (True, False)
    ),
)
@node_meta(
    id="StandardScaler",
    name="Standard Scaler",
    category="Preprocessing",
    description="Standardize features by removing the mean and scaling to unit variance.",
    params={"columns": [], "with_mean": True, "with_std": True},
    learns_from_data=True,
)
class StandardScalerCalculator(BaseCalculator):
    """Fit per-column statistics using sklearn locally or native Spark aggregates.

    Spark uses population variance (``ddof=0``), ignores null/NaN observations,
    and returns at most two statistics rows to the driver. Both options default
    to True. Empty explicit training selections produce a no-op artifact;
    selected features with zero training rows cannot be fitted.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return a schema with touched numeric columns promoted to ``float64``."""
        if not (config.get("with_mean", True) or config.get("with_std", True)):
            return input_schema
        return promote_configured_columns_to_float64(input_schema, config)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> StandardScalerArtifact:  # pylint: disable=arguments-differ
        """Dispatch the fit to the active engine and return the scaler-statistics artifact."""
        if user_picked_no_columns(config):
            return cast(StandardScalerArtifact, {})
        return cast(
            StandardScalerArtifact,
            fit_dual_engine(
                X,
                config,
                {
                    "polars": self._fit_polars,
                    "pandas": self._fit_pandas,
                    "spark": fit_spark_standard,
                },
            ),
        )

    @staticmethod
    def _fit_polars(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_polars(X, config)
        if not cols:
            return {}
        return _fit_standard(X_subset, cols, config)

    @staticmethod
    def _fit_pandas(X: Any, _y: Any, config: dict[str, Any]) -> dict[str, Any]:
        cols, X_subset = _select_subset_pandas(X, config)
        if not cols:
            return {}
        return _fit_standard(X_subset, cols, config)

fit(X, _y, config)

Dispatch the fit to the active engine and return the scaler-statistics artifact.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> StandardScalerArtifact:  # pylint: disable=arguments-differ
    """Dispatch the fit to the active engine and return the scaler-statistics artifact."""
    if user_picked_no_columns(config):
        return cast(StandardScalerArtifact, {})
    return cast(
        StandardScalerArtifact,
        fit_dual_engine(
            X,
            config,
            {
                "polars": self._fit_polars,
                "pandas": self._fit_pandas,
                "spark": fit_spark_standard,
            },
        ),
    )

infer_output_schema(input_schema, config)

Return a schema with touched numeric columns promoted to float64.

Source code in skyulf-core/skyulf/preprocessing/scaling/standard.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return a schema with touched numeric columns promoted to ``float64``."""
    if not (config.get("with_mean", True) or config.get("with_std", True)):
        return input_schema
    return promote_configured_columns_to_float64(input_schema, config)

StatefulTransformer

Fits + applies one pipeline step.

Accepts anything satisfying :class:~skyulf.core.protocols.CalculatorProtocol / :class:~skyulf.core.protocols.ApplierProtocol (structural typing) — a BaseCalculator/BaseApplier subclass, or any duck-typed object exposing matching fit/apply methods, works without subclassing.

Source code in skyulf-core/skyulf/preprocessing/base.py

class StatefulTransformer:
    """Fits + applies one pipeline step.

    Accepts anything satisfying :class:`~skyulf.core.protocols.CalculatorProtocol` /
    :class:`~skyulf.core.protocols.ApplierProtocol` (structural typing) — a
    ``BaseCalculator``/``BaseApplier`` subclass, or any duck-typed object
    exposing matching ``fit``/``apply`` methods, works without subclassing.
    """

    def __init__(
        self,
        calculator: CalculatorProtocol,
        applier: ApplierProtocol,
        node_id: str,
        apply_on_test: bool = True,
        apply_on_validation: bool = True,
    ):
        """Store the node pair and which held-out splits it should be applied to.

        ``apply_on_test``/``apply_on_validation`` turn the transform of those
        splits off without dropping them from the result. ``params`` starts
        empty and is filled by the first :meth:`fit_transform`; the profiling
        counters are zeroed here and overwritten on every run.
        """
        self.calculator = calculator
        self.applier = applier
        self.node_id = node_id
        self.apply_on_test = apply_on_test
        self.apply_on_validation = apply_on_validation
        self.params: dict[str, Any] = {}  # Store params in memory instead of ArtifactStore
        # Profiling metrics
        self.fit_time: float = 0.0
        self.peak_memory_bytes: int = 0
        self.rows_in: int = 0
        self.rows_out: int = 0
        self.train_sample_weight_: Any = None

    def fit_transform(
        self,
        dataset: SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple,
        config: dict[str, Any],
    ) -> SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple:
        """Fit on the training data, apply to the held-out splits, and profile the run.

        Wall time, row counts and peak memory land on ``self`` rather than in
        the return value, and are recorded in a ``finally`` so a failing node
        still reports what it cost. ``tracemalloc`` is borrowed, not owned: if
        the caller is already tracing, only the peak growth since entry can be
        attributed to this step and tracing is left running; otherwise it is
        started here and stopped again on exit.
        """
        validate_weighted_transformer(dataset, self.calculator, self.applier)
        self.rows_in, _ = get_data_stats(dataset)
        tracing_was_active = tracemalloc.is_tracing()
        if not tracing_was_active:
            tracemalloc.start()
            tracemalloc.reset_peak()
        peak_baseline = tracemalloc.get_traced_memory()[1] if tracemalloc.is_tracing() else 0
        # With caller-owned tracing we can only report new global peak growth since entry.
        self.peak_memory_bytes = 0
        self.rows_out = 0
        start = time.time()

        try:
            result = self._fit_transform_inner(dataset, config)
            self.rows_out, _ = get_data_stats(result)
            return result
        finally:
            self.fit_time = time.time() - start
            if tracemalloc.is_tracing():
                _, peak = tracemalloc.get_traced_memory()
                self.peak_memory_bytes = max(0, peak - peak_baseline)
            if not tracing_was_active and tracemalloc.is_tracing():
                tracemalloc.stop()

    def _fit_and_apply_training_data(
        self,
        data: Any,
        config: dict[str, Any],
        *,
        guard_split_output: bool = True,
    ) -> Any:
        """Fit one training input and return its train-time representation."""
        if isinstance(self.calculator, TrainTransformCalculatorProtocol):
            params, transformed = self.calculator.fit_transform_train(data, config)
            self.params = cast(dict[str, Any], params)
            if isinstance(transformed, SplitDataset):
                raise TypeError(
                    "Calculator returned SplitDataset from fit_transform_train, which is not supported."
                )
            return transformed

        self.params = cast(dict[str, Any], self.calculator.fit(data, config))
        if guard_split_output:
            return self._apply_guarded(data, self.params)
        return self.applier.apply(data, self.params)

    def fit_transform_weighted(self, data: Any, config: Any, weights: Any) -> tuple[Any, Any]:
        """Fit on real targets and propagate weights through the actual row selection."""
        if weights is None:
            return self._fit_and_apply_training_data(data, config), None
        validate_weighted_components(self.calculator, self.applier)
        weights = validate_sample_weight(weights, len(unpack_pipeline_input(data)[0]))
        resampling = import_module("skyulf.preprocessing.resampling")

        if type(self.calculator) in (
            resampling.OversamplingCalculator,
            resampling.UndersamplingCalculator,
        ):
            helper = import_module("skyulf.preprocessing._weighted_resampling")

            self.params, transformed, weights = helper.fit_resample_weighted(
                self.calculator, self.applier, data, config, weights
            )
            return transformed, weights
        if has_builtin_row_mapping(self.applier):
            self.params = cast(dict[str, Any], self.calculator.fit(data, config))
            return apply_weighted(
                self.applier, data, self.params | {"_history_training": True}, weights
            )
        if isinstance(self.calculator, TrainTransformCalculatorProtocol):
            validate_training_representation(self.calculator, self.applier)
            transformed = self._fit_and_apply_training_data(data, config)

            return transformed, validate_sample_weight(
                weights, len(unpack_pipeline_input(transformed)[0])
            )
        self.params = cast(dict[str, Any], self.calculator.fit(data, config))
        return apply_weighted(self.applier, data, self.params, weights)

    def _fit_transform_inner(
        self,
        dataset: SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple,
        config: dict[str, Any],
    ) -> SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple:
        # Check for DataFrame-like (Pandas, Polars, Wrapper)
        if (
            hasattr(dataset, "shape")
            and hasattr(dataset, "columns")
            and not isinstance(dataset, tuple)
        ):
            # Fit on the whole dataframe (be careful about leakage!)
            # ty can't narrow a Union through hasattr — cast once for both calls.
            frame = cast(Any, dataset)
            return self._fit_and_apply_training_data(frame, config, guard_split_output=False)

        # If dataset is a tuple (e.g. from FeatureTargetSplitter), pass it through.
        # This allows nodes like TrainTestSplitter to accept (X, y) tuples.
        if isinstance(dataset, tuple):
            return self._fit_and_apply_training_data(dataset, config, guard_split_output=False)

        # 1. Calculate on Train
        new_train, train_weight = self.fit_transform_weighted(
            dataset.train, config, dataset.train_sample_weight
        )

        # 2. Apply fitted params to held-out splits only
        new_test = dataset.test
        if self.apply_on_test:
            new_test = self._apply_guarded(dataset.test, self.params)

        new_val = dataset.validation
        if self.apply_on_validation and dataset.validation is not None:
            new_val = self._apply_guarded(dataset.validation, self.params)

        return SplitDataset(
            train=new_train,
            test=new_test,
            validation=new_val,
            train_sample_weight=train_weight,
            evaluation_coverage=self._evaluation_coverage(dataset, new_test, new_val),
        )

    def _evaluation_coverage(
        self, dataset: SplitDataset, new_test: Any, new_val: Any
    ) -> dict[str, dict[str, Any]]:
        """Carry held-out denominators across nodes without storing row-level data."""
        coverage = deepcopy(dataset.evaluation_coverage)
        for name, result in (("test", new_test), ("validation", new_val)):
            original = getattr(dataset, name)
            if original is None or result is None:
                continue
            before = len(unpack_pipeline_input(original)[0])
            after = len(unpack_pipeline_input(result)[0])
            previous = coverage.get(name, record_coverage(before, before))
            steps = list(previous.get("steps", []))
            if before != after:
                steps.append(
                    {"name": self.node_id, **record_coverage(before, after, step_name=self.node_id)}
                )
            coverage[name] = record_coverage(
                previous["input_rows"],
                after,
                steps,
                reason=previous.get("reason"),
                step_name=self.node_id,
            )
        return coverage

    def _apply_guarded(self, data: Any, params: dict[str, Any]) -> Any:
        """Apply the applier to `data` and raise if it produces a nested SplitDataset."""
        result = self.applier.apply(data, params)
        if isinstance(result, SplitDataset):
            raise TypeError(
                "Applier returned SplitDataset inside StatefulTransformer, which is not supported."
            )
        return result

    def _apply_to_split_dataset(
        self, dataset: SplitDataset, params: dict[str, Any]
    ) -> SplitDataset:
        """Apply the applier to each split (train/test/validation) of a SplitDataset."""
        validate_weighted_transformer(dataset, self.calculator, self.applier)
        if is_sampling_applier(self.applier):
            return dataset
        if dataset.train_sample_weight is None:
            new_train, train_weight = self._apply_guarded(dataset.train, params), None
        else:
            weights = validate_sample_weight(
                dataset.train_sample_weight, len(unpack_pipeline_input(cast(Any, dataset.train))[0])
            )
            new_train, train_weight = apply_weighted(self.applier, dataset.train, params, weights)

        new_test = dataset.test
        if self.apply_on_test:
            new_test = self._apply_guarded(dataset.test, params)

        new_val = dataset.validation
        if self.apply_on_validation and dataset.validation is not None:
            new_val = self._apply_guarded(dataset.validation, params)

        return SplitDataset(
            train=new_train,
            test=new_test,
            validation=new_val,
            train_sample_weight=train_weight,
            evaluation_coverage=self._evaluation_coverage(dataset, new_test, new_val),
        )

    def transform(
        self, dataset: SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple
    ) -> SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple:
        """Apply the params stored by the last fit, without refitting.

        Input detection mirrors :meth:`_fit_transform_inner` so a bare frame or
        an ``(X, y)`` tuple goes straight to the applier, while a
        ``SplitDataset`` is applied split-by-split honouring
        ``apply_on_test``/``apply_on_validation``. Built-in resamplers leave
        ``SplitDataset`` unchanged during replay, with or without weights.
        Calling this before any
        :meth:`fit_transform` hands the applier an empty params dict rather
        than raising.
        """
        # Use stored params
        params = self.params

        # Check for DataFrame-like (Pandas, Polars, Wrapper) input, mirroring
        # `_fit_transform_inner`'s detection above -- an `isinstance(dataset,
        # pd.DataFrame)`-only check here would misroute a raw (unwrapped)
        # `pl.DataFrame` into the SplitDataset branch below, crashing with
        # `AttributeError: 'DataFrame' object has no attribute 'train'`.
        if (
            hasattr(dataset, "shape")
            and hasattr(dataset, "columns")
            and not isinstance(dataset, tuple)
        ):
            return self.applier.apply(cast(Any, dataset), params)

        if isinstance(dataset, tuple):
            return self.applier.apply(dataset, params)

        # 2. Apply
        # ty can't narrow SplitDataset out of the SkyulfDataFrame branch of this
        # Union via isinstance alone (mirrors the `frame = cast(Any, dataset)`
        # note in `_fit_transform_inner` above).
        return self._apply_to_split_dataset(cast(SplitDataset, dataset), params)

__init__(calculator, applier, node_id, apply_on_test=True, apply_on_validation=True)

Store the node pair and which held-out splits it should be applied to.

apply_on_test/apply_on_validation turn the transform of those splits off without dropping them from the result. params starts empty and is filled by the first :meth:fit_transform; the profiling counters are zeroed here and overwritten on every run.

Source code in skyulf-core/skyulf/preprocessing/base.py

def __init__(
    self,
    calculator: CalculatorProtocol,
    applier: ApplierProtocol,
    node_id: str,
    apply_on_test: bool = True,
    apply_on_validation: bool = True,
):
    """Store the node pair and which held-out splits it should be applied to.

    ``apply_on_test``/``apply_on_validation`` turn the transform of those
    splits off without dropping them from the result. ``params`` starts
    empty and is filled by the first :meth:`fit_transform`; the profiling
    counters are zeroed here and overwritten on every run.
    """
    self.calculator = calculator
    self.applier = applier
    self.node_id = node_id
    self.apply_on_test = apply_on_test
    self.apply_on_validation = apply_on_validation
    self.params: dict[str, Any] = {}  # Store params in memory instead of ArtifactStore
    # Profiling metrics
    self.fit_time: float = 0.0
    self.peak_memory_bytes: int = 0
    self.rows_in: int = 0
    self.rows_out: int = 0
    self.train_sample_weight_: Any = None

fit_transform(dataset, config)

Fit on the training data, apply to the held-out splits, and profile the run.

Wall time, row counts and peak memory land on self rather than in the return value, and are recorded in a finally so a failing node still reports what it cost. tracemalloc is borrowed, not owned: if the caller is already tracing, only the peak growth since entry can be attributed to this step and tracing is left running; otherwise it is started here and stopped again on exit.

Source code in skyulf-core/skyulf/preprocessing/base.py

def fit_transform(
    self,
    dataset: SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple,
    config: dict[str, Any],
) -> SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple:
    """Fit on the training data, apply to the held-out splits, and profile the run.

    Wall time, row counts and peak memory land on ``self`` rather than in
    the return value, and are recorded in a ``finally`` so a failing node
    still reports what it cost. ``tracemalloc`` is borrowed, not owned: if
    the caller is already tracing, only the peak growth since entry can be
    attributed to this step and tracing is left running; otherwise it is
    started here and stopped again on exit.
    """
    validate_weighted_transformer(dataset, self.calculator, self.applier)
    self.rows_in, _ = get_data_stats(dataset)
    tracing_was_active = tracemalloc.is_tracing()
    if not tracing_was_active:
        tracemalloc.start()
        tracemalloc.reset_peak()
    peak_baseline = tracemalloc.get_traced_memory()[1] if tracemalloc.is_tracing() else 0
    # With caller-owned tracing we can only report new global peak growth since entry.
    self.peak_memory_bytes = 0
    self.rows_out = 0
    start = time.time()

    try:
        result = self._fit_transform_inner(dataset, config)
        self.rows_out, _ = get_data_stats(result)
        return result
    finally:
        self.fit_time = time.time() - start
        if tracemalloc.is_tracing():
            _, peak = tracemalloc.get_traced_memory()
            self.peak_memory_bytes = max(0, peak - peak_baseline)
        if not tracing_was_active and tracemalloc.is_tracing():
            tracemalloc.stop()

fit_transform_weighted(data, config, weights)

Fit on real targets and propagate weights through the actual row selection.

Source code in skyulf-core/skyulf/preprocessing/base.py

def fit_transform_weighted(self, data: Any, config: Any, weights: Any) -> tuple[Any, Any]:
    """Fit on real targets and propagate weights through the actual row selection."""
    if weights is None:
        return self._fit_and_apply_training_data(data, config), None
    validate_weighted_components(self.calculator, self.applier)
    weights = validate_sample_weight(weights, len(unpack_pipeline_input(data)[0]))
    resampling = import_module("skyulf.preprocessing.resampling")

    if type(self.calculator) in (
        resampling.OversamplingCalculator,
        resampling.UndersamplingCalculator,
    ):
        helper = import_module("skyulf.preprocessing._weighted_resampling")

        self.params, transformed, weights = helper.fit_resample_weighted(
            self.calculator, self.applier, data, config, weights
        )
        return transformed, weights
    if has_builtin_row_mapping(self.applier):
        self.params = cast(dict[str, Any], self.calculator.fit(data, config))
        return apply_weighted(
            self.applier, data, self.params | {"_history_training": True}, weights
        )
    if isinstance(self.calculator, TrainTransformCalculatorProtocol):
        validate_training_representation(self.calculator, self.applier)
        transformed = self._fit_and_apply_training_data(data, config)

        return transformed, validate_sample_weight(
            weights, len(unpack_pipeline_input(transformed)[0])
        )
    self.params = cast(dict[str, Any], self.calculator.fit(data, config))
    return apply_weighted(self.applier, data, self.params, weights)

transform(dataset)

Apply the params stored by the last fit, without refitting.

Input detection mirrors :meth:_fit_transform_inner so a bare frame or an (X, y) tuple goes straight to the applier, while a SplitDataset is applied split-by-split honouring apply_on_test/apply_on_validation. Built-in resamplers leave SplitDataset unchanged during replay, with or without weights. Calling this before any :meth:fit_transform hands the applier an empty params dict rather than raising.

Source code in skyulf-core/skyulf/preprocessing/base.py

def transform(
    self, dataset: SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple
) -> SplitDataset | pd.DataFrame | pl.DataFrame | SkyulfDataFrame | tuple:
    """Apply the params stored by the last fit, without refitting.

    Input detection mirrors :meth:`_fit_transform_inner` so a bare frame or
    an ``(X, y)`` tuple goes straight to the applier, while a
    ``SplitDataset`` is applied split-by-split honouring
    ``apply_on_test``/``apply_on_validation``. Built-in resamplers leave
    ``SplitDataset`` unchanged during replay, with or without weights.
    Calling this before any
    :meth:`fit_transform` hands the applier an empty params dict rather
    than raising.
    """
    # Use stored params
    params = self.params

    # Check for DataFrame-like (Pandas, Polars, Wrapper) input, mirroring
    # `_fit_transform_inner`'s detection above -- an `isinstance(dataset,
    # pd.DataFrame)`-only check here would misroute a raw (unwrapped)
    # `pl.DataFrame` into the SplitDataset branch below, crashing with
    # `AttributeError: 'DataFrame' object has no attribute 'train'`.
    if (
        hasattr(dataset, "shape")
        and hasattr(dataset, "columns")
        and not isinstance(dataset, tuple)
    ):
        return self.applier.apply(cast(Any, dataset), params)

    if isinstance(dataset, tuple):
        return self.applier.apply(dataset, params)

    # 2. Apply
    # ty can't narrow SplitDataset out of the SkyulfDataFrame branch of this
    # Union via isinstance alone (mirrors the `frame = cast(Any, dataset)`
    # note in `_fit_transform_inner` above).
    return self._apply_to_split_dataset(cast(SplitDataset, dataset), params)

TargetEncoderApplier

Bases: BaseApplier

Replace categorical columns with the target statistics the fitted encoder learned.

Both engines transform through the same sklearn TargetEncoder carried in the artifact, so parity comes from sharing it. The output shape follows the target type: binary and regression targets are encoded in place, while a multiclass target drops the originals and emits one {col}_cls{i} column per class. Multiclass output names must not collide with retained columns; conflicts raise ValueError before any input values are replaced.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

class TargetEncoderApplier(BaseApplier):
    """Replace categorical columns with the target statistics the fitted encoder learned.

    Both engines transform through the same sklearn ``TargetEncoder`` carried in
    the artifact, so parity comes from sharing it. The output shape follows the
    target type: binary and regression targets are encoded in place, while a
    multiclass target drops the originals and emits one ``{col}_cls{i}`` column
    per class. Multiclass output names must not collide with retained columns;
    conflicts raise ``ValueError`` before any input values are replaced.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect the saved full-data encoder while leaving cross-fitted training values alone."""
        if not local_state_fields(
            raw, "target_encoder", {"type", "columns", "encoder_object"}, allow_empty=True
        ):
            return raw
        columns = raw["columns"]
        if not isinstance(columns, (list, tuple)) or any(not isinstance(c, str) for c in columns):
            raise ValueError("Fitted target encoder columns must be ordered strings.")
        encoder = raw["encoder_object"]
        required = {
            "n_features_in_",
            "categories_",
            "target_type_",
            "classes_",
            "target_mean_",
            "encodings_",
        }
        if type(encoder) is not TargetEncoder or not required.issubset(vars(encoder)):
            raise ValueError("Expected a fitted TargetEncoder.")
        if any(callable(value) for value in vars(encoder).values()):
            raise ValueError("Overridden target estimator methods are unsupported.")
        _validate_target_categories(encoder, len(columns))
        _validate_target_statistics(encoder)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved target-statistic lookups without refitting or admitting workers."""
        if engine not in ("pandas", "polars"):
            return None
        TargetEncoderApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch to the engine-specific transform, forwarding ``(X, y)`` only when present."""
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {"polars": _target_apply_polars, "pandas": _target_apply_pandas},
        )

apply(X, y, params)

Dispatch to the engine-specific transform, forwarding (X, y) only when present.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch to the engine-specific transform, forwarding ``(X, y)`` only when present."""
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {"polars": _target_apply_polars, "pandas": _target_apply_pandas},
    )

inference_capability(state, *, engine) staticmethod

Describe saved target-statistic lookups without refitting or admitting workers.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved target-statistic lookups without refitting or admitting workers."""
    if engine not in ("pandas", "polars"):
        return None
    TargetEncoderApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect the saved full-data encoder while leaving cross-fitted training values alone.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect the saved full-data encoder while leaving cross-fitted training values alone."""
    if not local_state_fields(
        raw, "target_encoder", {"type", "columns", "encoder_object"}, allow_empty=True
    ):
        return raw
    columns = raw["columns"]
    if not isinstance(columns, (list, tuple)) or any(not isinstance(c, str) for c in columns):
        raise ValueError("Fitted target encoder columns must be ordered strings.")
    encoder = raw["encoder_object"]
    required = {
        "n_features_in_",
        "categories_",
        "target_type_",
        "classes_",
        "target_mean_",
        "encodings_",
    }
    if type(encoder) is not TargetEncoder or not required.issubset(vars(encoder)):
        raise ValueError("Expected a fitted TargetEncoder.")
    if any(callable(value) for value in vars(encoder).values()):
        raise ValueError("Overridden target estimator methods are unsupported.")
    _validate_target_categories(encoder, len(columns))
    _validate_target_statistics(encoder)
    return raw

TargetEncoderCalculator

Bases: BaseCalculator

Fit a sklearn TargetEncoder against y, cross-fitting the training rows.

A target is mandatory: with no y the node logs a warning and returns an empty artifact rather than failing. The target column is excluded from the encoded set, since encoding it against itself is degenerate. This calculator also satisfies TrainTransformCalculatorProtocol, so the training split receives out-of-fold encodings — a row's own target never leaks into its feature — with the fold count capped by the smallest class count, while held-out splits go through the plain applier.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

@NodeRegistry.register("TargetEncoder", TargetEncoderApplier)
@node_meta(
    id="TargetEncoder",
    name="Target Encoder",
    category="Preprocessing",
    description="Encode categorical features using target statistics.",
    params={"smooth": "auto", "target_type": "auto", "columns": []},
    learns_from_data=True,
)
class TargetEncoderCalculator(BaseCalculator):
    """Fit a sklearn ``TargetEncoder`` against ``y``, cross-fitting the training rows.

    A target is mandatory: with no ``y`` the node logs a warning and returns an
    empty artifact rather than failing. The target column is excluded from the
    encoded set, since encoding it against itself is degenerate. This calculator
    also satisfies ``TrainTransformCalculatorProtocol``, so the training split
    receives out-of-fold encodings — a row's own target never leaks into its
    feature — with the fold count capped by the smallest class count, while
    held-out splits go through the plain applier.
    """

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> TargetEncoderArtifact:  # pylint: disable=arguments-differ
        """Short-circuit an explicit empty column selection, else fit on the frame's own engine."""
        if user_picked_no_columns(config):
            return {}
        return cast(
            TargetEncoderArtifact,
            fit_dual_engine(
                (X, y) if y is not None else X,
                config,
                {"polars": _target_fit_polars, "pandas": _target_fit_pandas},
            ),
        )

    def fit_transform_train(
        self, df: pd.DataFrame | SkyulfDataFrame | tuple, config: dict[str, Any]
    ) -> tuple[TargetEncoderArtifact, Any]:
        """Fit sklearn TargetEncoder and cross-fit the pipeline training rows."""
        if user_picked_no_columns(config):
            return {}, df

        artifact, transformed = fit_transform_train_dual_engine(
            df,
            config,
            {
                "polars": _target_fit_transform_train_polars,
                "pandas": _target_fit_transform_train_pandas,
            },
        )
        return cast(
            TargetEncoderArtifact,
            artifact,
        ), transformed

    def infer_output_schema(
        self,
        input_schema: SkyulfSchema,
        config: dict[str, Any],
    ) -> SkyulfSchema | None:
        """Predict the schema only when ``target_type`` is explicitly binary or continuous.

        Anything else — ``"auto"`` included — can resolve to multiclass at fit
        time, which changes the column set in a data-dependent way, so ``None``
        is returned and callers fall back to runtime introspection.
        """
        # For binary/continuous targets, the encoder replaces values in
        # source columns in place — same column names, dtype becomes float
        # (per-column dtype is best-effort so we don't bother rewriting it).
        #
        # For multiclass targets, the apply logic (see
        # ``_target_apply_polars``/``_target_apply_pandas``) drops the
        # original columns and creates ``{col}_cls{i}`` columns instead — the
        # number of classes is data-dependent and unknown here, so we can't
        # confidently predict the output columns. The default/"auto"
        # target_type is resolved to multiclass at fit time whenever y has
        # more than two classes, so we must also treat "auto" as unknown
        # rather than assuming binary/continuous. Only the explicit
        # "binary"/"continuous" config values are confidently in-place.
        #
        # "continuous" is sklearn's name for a regression target. The value is
        # forwarded to ``TargetEncoder(target_type=...)`` verbatim and its
        # accepted set is {auto, binary, multiclass, continuous}, so the more
        # natural-sounding "regression" raises InvalidParameterError at fit and
        # could never reach this check through a working config.
        if config.get("target_type", "auto") not in ("binary", "continuous"):
            return None
        return input_schema

fit(X, y, config)

Short-circuit an explicit empty column selection, else fit on the frame's own engine.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> TargetEncoderArtifact:  # pylint: disable=arguments-differ
    """Short-circuit an explicit empty column selection, else fit on the frame's own engine."""
    if user_picked_no_columns(config):
        return {}
    return cast(
        TargetEncoderArtifact,
        fit_dual_engine(
            (X, y) if y is not None else X,
            config,
            {"polars": _target_fit_polars, "pandas": _target_fit_pandas},
        ),
    )

fit_transform_train(df, config)

Fit sklearn TargetEncoder and cross-fit the pipeline training rows.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

def fit_transform_train(
    self, df: pd.DataFrame | SkyulfDataFrame | tuple, config: dict[str, Any]
) -> tuple[TargetEncoderArtifact, Any]:
    """Fit sklearn TargetEncoder and cross-fit the pipeline training rows."""
    if user_picked_no_columns(config):
        return {}, df

    artifact, transformed = fit_transform_train_dual_engine(
        df,
        config,
        {
            "polars": _target_fit_transform_train_polars,
            "pandas": _target_fit_transform_train_pandas,
        },
    )
    return cast(
        TargetEncoderArtifact,
        artifact,
    ), transformed

infer_output_schema(input_schema, config)

Predict the schema only when target_type is explicitly binary or continuous.

Anything else — "auto" included — can resolve to multiclass at fit time, which changes the column set in a data-dependent way, so None is returned and callers fall back to runtime introspection.

Source code in skyulf-core/skyulf/preprocessing/encoding/target.py

def infer_output_schema(
    self,
    input_schema: SkyulfSchema,
    config: dict[str, Any],
) -> SkyulfSchema | None:
    """Predict the schema only when ``target_type`` is explicitly binary or continuous.

    Anything else — ``"auto"`` included — can resolve to multiclass at fit
    time, which changes the column set in a data-dependent way, so ``None``
    is returned and callers fall back to runtime introspection.
    """
    # For binary/continuous targets, the encoder replaces values in
    # source columns in place — same column names, dtype becomes float
    # (per-column dtype is best-effort so we don't bother rewriting it).
    #
    # For multiclass targets, the apply logic (see
    # ``_target_apply_polars``/``_target_apply_pandas``) drops the
    # original columns and creates ``{col}_cls{i}`` columns instead — the
    # number of classes is data-dependent and unknown here, so we can't
    # confidently predict the output columns. The default/"auto"
    # target_type is resolved to multiclass at fit time whenever y has
    # more than two classes, so we must also treat "auto" as unknown
    # rather than assuming binary/continuous. Only the explicit
    # "binary"/"continuous" config values are confidently in-place.
    #
    # "continuous" is sklearn's name for a regression target. The value is
    # forwarded to ``TargetEncoder(target_type=...)`` verbatim and its
    # accepted set is {auto, binary, multiclass, continuous}, so the more
    # natural-sounding "regression" raises InvalidParameterError at fit and
    # could never reach this check through a working config.
    if config.get("target_type", "auto") not in ("binary", "continuous"):
        return None
    return input_schema

TextCleaningApplier

Bases: BaseApplier

Apply an ordered chain of text operations to the resolved columns.

The pandas and polars paths must agree value-for-value. Each column is coerced to string first, then every entry in operations is applied in list order so later operations see earlier results. The pandas coercion masks nulls back afterwards, otherwise astype(str) would turn them into the literal "nan". Unrecognized operation names raise ValueError, and an empty column set or empty operation list is a no-op.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

class TextCleaningApplier(BaseApplier):
    """Apply an ordered chain of text operations to the resolved columns.

    The pandas and polars paths must agree value-for-value. Each column is
    coerced to string first, then every entry in ``operations`` is applied in
    list order so later operations see earlier results. The pandas coercion
    masks nulls back afterwards, otherwise ``astype(str)`` would turn them into
    the literal ``"nan"``. Unrecognized operation names raise ``ValueError``, and
    an empty column set or empty operation list is a no-op.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved operations without rewriting their order, defaults or no-ops."""
        if not local_state_fields(
            raw, "text_cleaning", {"type", "columns", "operations"}, allow_empty=True
        ):
            return raw
        _columns(raw["columns"])
        operations = raw["operations"]
        if operations is None:
            return raw
        if type(operations) not in (list, tuple):
            raise ValueError("Fitted text operations must be a list or tuple.")
        _validate_text_operations(operations)
        for operation in operations:
            _validate_inference_text_operation(operation)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local row operations while retaining engine-specific regex behavior."""
        if engine not in ("pandas", "polars"):
            return None
        TextCleaningApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch the operation chain to the pandas or polars path; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        operations = params.get("operations", [])
        valid = resolve_valid_columns(X, cols)
        if not valid or not operations:
            return X, _y
        _validate_text_operations(operations)

        exprs = []
        for col in valid:
            expr = pl.col(col).cast(pl.String)
            for op in operations:
                handler = _TEXT_OPS_POLARS[op["op"]]
                expr = handler(expr, op)
            exprs.append(expr.alias(col))
        return X.with_columns(exprs), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        cols = params.get("columns", [])
        operations = params.get("operations", [])
        valid = resolve_valid_columns(X, cols)
        if not valid or not operations:
            return X, _y
        _validate_text_operations(operations)

        df_out = X.copy()
        for col in valid:
            if not pd.api.types.is_string_dtype(df_out[col]):
                null_mask = df_out[col].isna()
                df_out[col] = df_out[col].astype(str).mask(null_mask)
            series = df_out[col]
            for op in operations:
                handler = _TEXT_OPS_PANDAS[op["op"]]
                series = handler(series, op)
            df_out[col] = series
        return df_out, _y

apply(X, _y, params)

Dispatch the operation chain to the pandas or polars path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch the operation chain to the pandas or polars path; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local row operations while retaining engine-specific regex behavior.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local row operations while retaining engine-specific regex behavior."""
    if engine not in ("pandas", "polars"):
        return None
    TextCleaningApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved operations without rewriting their order, defaults or no-ops.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved operations without rewriting their order, defaults or no-ops."""
    if not local_state_fields(
        raw, "text_cleaning", {"type", "columns", "operations"}, allow_empty=True
    ):
        return raw
    _columns(raw["columns"])
    operations = raw["operations"]
    if operations is None:
        return raw
    if type(operations) not in (list, tuple):
        raise ValueError("Fitted text operations must be a list or tuple.")
    _validate_text_operations(operations)
    for operation in operations:
        _validate_inference_text_operation(operation)
    return raw

TextCleaningCalculator

Bases: BaseCalculator

Resolve text-cleaning config into an artifact; nothing is learned from data.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

@NodeRegistry.register("TextCleaning", TextCleaningApplier)
@node_meta(
    id="TextCleaning",
    name="Text Cleaning",
    category="Cleaning",
    description="Clean text data (trim, case conversion, remove special chars).",
    params={"columns": [], "operations": []},
    learns_from_data=False,
)
class TextCleaningCalculator(BaseCalculator):
    """Resolve text-cleaning config into an artifact; nothing is learned from data."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Pass the input schema through: string values are rewritten in place."""
        # Text cleaning rewrites string values in place; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TextCleaningArtifact:  # pylint: disable=arguments-differ
        """Build the artifact, auto-detecting text columns when none are named.

        Two cases short-circuit to an empty artifact: an explicit
        ``columns: []`` from the UI, and auto-detection finding no text columns
        at all. Both make the applier a no-op rather than coercing numeric
        columns to text.
        """
        if user_picked_no_columns(config):
            return {}
        cols = resolve_columns(X, config, _auto_detect_text_columns)
        if not cols:
            return {}
        return {
            "type": "text_cleaning",
            "columns": cols,
            "operations": config.get("operations", []),
        }

fit(X, _y, config)

Build the artifact, auto-detecting text columns when none are named.

Two cases short-circuit to an empty artifact: an explicit columns: [] from the UI, and auto-detection finding no text columns at all. Both make the applier a no-op rather than coercing numeric columns to text.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TextCleaningArtifact:  # pylint: disable=arguments-differ
    """Build the artifact, auto-detecting text columns when none are named.

    Two cases short-circuit to an empty artifact: an explicit
    ``columns: []`` from the UI, and auto-detection finding no text columns
    at all. Both make the applier a no-op rather than coercing numeric
    columns to text.
    """
    if user_picked_no_columns(config):
        return {}
    cols = resolve_columns(X, config, _auto_detect_text_columns)
    if not cols:
        return {}
    return {
        "type": "text_cleaning",
        "columns": cols,
        "operations": config.get("operations", []),
    }

infer_output_schema(input_schema, config)

Pass the input schema through: string values are rewritten in place.

Source code in skyulf-core/skyulf/preprocessing/cleaning/text.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Pass the input schema through: string values are rewritten in place."""
    # Text cleaning rewrites string values in place; column set is preserved.
    return input_schema

TfidfVectorizerApplier

Bases: BaseApplier

Attach one TF-IDF-weighted column per vocabulary term to the frame.

The configured text columns are joined into a single corpus string before transforming, the dense result is concatenated on, and the source columns survive unless drop_original is set. This node uses the text-specific dispatcher: the polars path runs natively, so only the text payload crosses into sklearn, and it falls back to a full pandas round-trip when a text column is not String dtype.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

class TfidfVectorizerApplier(BaseApplier):
    """Attach one TF-IDF-weighted column per vocabulary term to the frame.

    The configured text columns are joined into a single corpus string before
    transforming, the dense result is concatenated on, and the source columns
    survive unless ``drop_original`` is set. This node uses the text-specific
    dispatcher: the polars path runs natively, so only the text payload crosses
    into sklearn, and it falls back to a full pandas round-trip when a text
    column is not String dtype.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned vocabulary and IDF weights without recomputing corpus statistics."""
        vectorizer = _validate_local_vectorizer(
            raw, "tfidf_vectorizer", TfidfVectorizer, {"vocabulary", "max_features", "idf"}
        )
        if vectorizer is not None:
            _validate_vectorizer_vocabulary(raw, vectorizer)
            _validate_saved_idf(raw, vectorizer)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe fixed document weighting while leaving arbitrary analyzers undeclared."""
        if engine not in ("pandas", "polars"):
            return None
        TfidfVectorizerApplier.validate_inference_state(state)
        if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
        return apply_text_dual_engine(
            X, params, _tfidf_apply_pandas, _sklearn_vectorizer_apply_polars
        )

apply(X, _y, params)

Dispatch through the text-specific dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
    return apply_text_dual_engine(
        X, params, _tfidf_apply_pandas, _sklearn_vectorizer_apply_polars
    )

inference_capability(state, *, engine) staticmethod

Describe fixed document weighting while leaving arbitrary analyzers undeclared.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe fixed document weighting while leaving arbitrary analyzers undeclared."""
    if engine not in ("pandas", "polars"):
        return None
    TfidfVectorizerApplier.validate_inference_state(state)
    if state and _vectorizer_uses_callbacks(state["vectorizer_object"]):
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect learned vocabulary and IDF weights without recomputing corpus statistics.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned vocabulary and IDF weights without recomputing corpus statistics."""
    vectorizer = _validate_local_vectorizer(
        raw, "tfidf_vectorizer", TfidfVectorizer, {"vocabulary", "max_features", "idf"}
    )
    if vectorizer is not None:
        _validate_vectorizer_vocabulary(raw, vectorizer)
        _validate_saved_idf(raw, vectorizer)
    return raw

TfidfVectorizerCalculator

Bases: BaseCalculator

Fit a TfidfVectorizer vocabulary and IDF weights on the joined text.

Both the vocabulary and the per-term IDF weights are learned from the training corpus and carried in the artifact, so the weights stay inspectable and serialisable. A very wide output is warned about rather than raising, since it is a memory problem, not an error.

Schema inference inherits the base None result: the learned vocabulary determines the output columns, so their schema is only known at runtime.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

@NodeRegistry.register("tfidf_vectorizer", TfidfVectorizerApplier)
@node_meta(
    id="tfidf_vectorizer",
    name="TF-IDF Vectorizer",
    category="Text",
    description=(
        "Convert text columns to TF-IDF weighted feature columns. "
        "Penalises very common tokens and rewards rare-but-informative ones."
    ),
    params={
        "columns": [],
        "max_features": None,
        "min_df": 1,
        "max_df": 1.0,
        "ngram_range": [1, 1],
        "sublinear_tf": False,
        "lowercase": True,
        "stop_words": None,
        "drop_original": False,
    },
    tags=["text", "nlp", "tfidf", "vectorizer"],
    learns_from_data=True,
)
class TfidfVectorizerCalculator(BaseCalculator):
    """Fit a ``TfidfVectorizer`` vocabulary and IDF weights on the joined text.

    Both the vocabulary and the per-term IDF weights are learned from the
    training corpus and carried in the artifact, so the weights stay inspectable
    and serialisable. A very wide output is warned about rather than raising,
    since it is a memory problem, not an error.

    Schema inference inherits the base ``None`` result: the learned vocabulary
    determines the output columns, so their schema is only known at runtime.
    """

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TfidfVectorizerArtifact:  # pylint: disable=arguments-differ
        """Narrow to the configured text columns and fit the vocabulary on their joined text.

        An empty or wholly-absent column selection yields an empty artifact so
        the applier no-ops. An empty vocabulary also yields an empty artifact
        with a warning; source columns are retained even with ``drop_original``.
        """
        resolved = resolve_fit_text_columns(X, config, _y)
        if resolved is None:
            return {}
        text_frame, valid_cols = resolved

        artifact = _build_tfidf_artifact(config, text_frame, valid_cols)
        validate_text_output_names(X, artifact, valid_cols)
        return artifact

fit(X, _y, config)

Narrow to the configured text columns and fit the vocabulary on their joined text.

An empty or wholly-absent column selection yields an empty artifact so the applier no-ops. An empty vocabulary also yields an empty artifact with a warning; source columns are retained even with drop_original.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tfidf_vectorizer.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TfidfVectorizerArtifact:  # pylint: disable=arguments-differ
    """Narrow to the configured text columns and fit the vocabulary on their joined text.

    An empty or wholly-absent column selection yields an empty artifact so
    the applier no-ops. An empty vocabulary also yields an empty artifact
    with a warning; source columns are retained even with ``drop_original``.
    """
    resolved = resolve_fit_text_columns(X, config, _y)
    if resolved is None:
        return {}
    text_frame, valid_cols = resolved

    artifact = _build_tfidf_artifact(config, text_frame, valid_cols)
    validate_text_output_names(X, artifact, valid_cols)
    return artifact

TokenizerApplier

Bases: BaseApplier

Emit one space-joined token string per source column, not one column per token.

Keeping the tokens in a single {col}__tokens string is deliberate: it stays inspectable and can feed a downstream vectorizer, whereas exploding to columns would make the output width data-dependent. add_token_count adds a companion {col}__token_count integer column. Nulls are coerced to empty strings before tokenizing, and sources survive unless drop_original is set. The node is stateless — the sklearn analyzer is rebuilt from params on every call. The polars path runs natively and falls back to pandas when a text column is not String dtype.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

class TokenizerApplier(BaseApplier):
    """Emit one space-joined token string per source column, not one column per token.

    Keeping the tokens in a single ``{col}__tokens`` string is deliberate: it
    stays inspectable and can feed a downstream vectorizer, whereas exploding to
    columns would make the output width data-dependent. ``add_token_count`` adds
    a companion ``{col}__token_count`` integer column. Nulls are coerced to empty
    strings before tokenizing, and sources survive unless ``drop_original`` is
    set. The node is stateless — the sklearn analyzer is rebuilt from ``params``
    on every call. The polars path runs natively and falls back to pandas when a
    text column is not String dtype.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved analyzer configuration without constructing or executing it."""
        fields = {
            "type",
            "columns",
            "analyzer",
            "lowercase",
            "stop_words",
            "ngram_range",
            "output_columns",
            "add_token_count",
            "drop_original",
        }
        if not local_state_fields(raw, "tokenizer", fields, allow_empty=True):
            return raw
        _validate_text_layout(raw)
        _validate_text_settings(raw["lowercase"], raw["stop_words"], raw["ngram_range"])
        if raw["add_token_count"] is not None:
            local_boolean(raw["add_token_count"], "add_token_count")
        if not callable(raw["analyzer"]) and raw["analyzer"] not in ("word", "char", "char_wb"):
            raise ValueError("Fitted tokenizer analyzer is unknown.")
        expected = _build_tokenizer_artifact(raw, raw["columns"])["output_columns"]
        if list(raw["output_columns"]) != expected:
            raise ValueError("Fitted tokenizer output names disagree with its selected columns.")
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe per-document tokenization without promising arbitrary callback behavior."""
        if engine not in ("pandas", "polars"):
            return None
        TokenizerApplier.validate_inference_state(state)
        if state and callable(state["analyzer"]):
            return None
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
        return apply_text_dual_engine(X, params, _tokenizer_apply_pandas, _tokenizer_apply_polars)

apply(X, _y, params)

Dispatch through the text-specific dual-engine path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch through the text-specific dual-engine path; ``y`` passes through."""
    return apply_text_dual_engine(X, params, _tokenizer_apply_pandas, _tokenizer_apply_polars)

inference_capability(state, *, engine) staticmethod

Describe per-document tokenization without promising arbitrary callback behavior.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe per-document tokenization without promising arbitrary callback behavior."""
    if engine not in ("pandas", "polars"):
        return None
    TokenizerApplier.validate_inference_state(state)
    if state and callable(state["analyzer"]):
        return None
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved analyzer configuration without constructing or executing it.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved analyzer configuration without constructing or executing it."""
    fields = {
        "type",
        "columns",
        "analyzer",
        "lowercase",
        "stop_words",
        "ngram_range",
        "output_columns",
        "add_token_count",
        "drop_original",
    }
    if not local_state_fields(raw, "tokenizer", fields, allow_empty=True):
        return raw
    _validate_text_layout(raw)
    _validate_text_settings(raw["lowercase"], raw["stop_words"], raw["ngram_range"])
    if raw["add_token_count"] is not None:
        local_boolean(raw["add_token_count"], "add_token_count")
    if not callable(raw["analyzer"]) and raw["analyzer"] not in ("word", "char", "char_wb"):
        raise ValueError("Fitted tokenizer analyzer is unknown.")
    expected = _build_tokenizer_artifact(raw, raw["columns"])["output_columns"]
    if list(raw["output_columns"]) != expected:
        raise ValueError("Fitted tokenizer output names disagree with its selected columns.")
    return raw

TokenizerCalculator

Bases: BaseCalculator

Record the resolved text columns and the tokenizer settings — nothing is fitted.

learns_from_data=False: the artifact is pure configuration, so it is reproducible across runs and safe to reuse on data the calculator never saw.

Schema inference conservatively inherits the base None result. Fitting resolves source columns and excludes the target, including a separately supplied y; runtime introspection determines which source and token columns remain after applying drop_original.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

@NodeRegistry.register("tokenizer", TokenizerApplier)
@node_meta(
    id="tokenizer",
    name="Tokenizer",
    category="Text",
    description=(
        "Split text columns into tokens (word, char, or char_wb). "
        "Outputs a space-joined token string column per source column, "
        "optionally with a token-count column. Stateless — no vocabulary fitted. "
        "Inspection / intermediate tool only: do NOT chain before a vectorizer "
        "(Count / TF-IDF / Hashing already tokenize internally)."
    ),
    params={
        "columns": [],
        "analyzer": "word",
        "lowercase": True,
        "stop_words": None,
        "ngram_range": [1, 1],
        "add_token_count": False,
        "drop_original": False,
    },
    tags=["text", "nlp", "tokenizer"],
    learns_from_data=False,
)
class TokenizerCalculator(BaseCalculator):
    """Record the resolved text columns and the tokenizer settings — nothing is fitted.

    ``learns_from_data=False``: the artifact is pure configuration, so it is
    reproducible across runs and safe to reuse on data the calculator never saw.

    Schema inference conservatively inherits the base ``None`` result. Fitting
    resolves source columns and excludes the target, including a separately
    supplied ``y``; runtime introspection determines which source and token
    columns remain after applying ``drop_original``.
    """

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TokenizerArtifact:  # pylint: disable=arguments-differ
        """Resolve the configured text column names and record them with the tokenizer settings.

        Uses the column-names-only resolver rather than the data-narrowing
        sibling, so a polars input is never converted to pandas just to fit. An
        empty or wholly-absent selection yields an empty artifact so the applier
        no-ops.
        """
        valid_cols = resolve_fit_text_valid_columns(X, config, _y)
        if valid_cols is None:
            return {}

        artifact = _build_tokenizer_artifact(config, valid_cols)
        validate_text_output_names(X, artifact, valid_cols)
        return artifact

fit(X, _y, config)

Resolve the configured text column names and record them with the tokenizer settings.

Uses the column-names-only resolver rather than the data-narrowing sibling, so a polars input is never converted to pandas just to fit. An empty or wholly-absent selection yields an empty artifact so the applier no-ops.

Source code in skyulf-core/skyulf/preprocessing/vectorization/tokenizer.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> TokenizerArtifact:  # pylint: disable=arguments-differ
    """Resolve the configured text column names and record them with the tokenizer settings.

    Uses the column-names-only resolver rather than the data-narrowing
    sibling, so a polars input is never converted to pandas just to fit. An
    empty or wholly-absent selection yields an empty artifact so the applier
    no-ops.
    """
    valid_cols = resolve_fit_text_valid_columns(X, config, _y)
    if valid_cols is None:
        return {}

    artifact = _build_tokenizer_artifact(config, valid_cols)
    validate_text_output_names(X, artifact, valid_cols)
    return artifact

UndersamplingApplier

Bases: BaseApplier

Shrink the majority classes with the imblearn under-sampler named in the artifact.

Sees the training split only: :mod:.pipeline excludes the resampling nodes from apply_on_test/apply_on_validation. That guard matters more here than for oversampling, because these samplers delete real rows rather than synthesise new ones.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

class UndersamplingApplier(BaseApplier):
    """Shrink the majority classes with the imblearn under-sampler named in the artifact.

    Sees the training split only: :mod:`.pipeline` excludes the resampling nodes
    from ``apply_on_test``/``apply_on_validation``. That guard matters more here
    than for oversampling, because these samplers delete real rows rather than
    synthesise new ones.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved selection settings without loading the optional sampler package."""
        return _validate_resampling_state(raw, "undersampling", "random_under_sampling")

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Require full class context; prediction still skips every sampling method."""
        if engine not in ("pandas", "polars"):
            return None
        UndersamplingApplier.validate_inference_state(state)
        if not _under_context_known(state):
            return None
        return ExecutionCapability(engine, "apply", "local", "filter", "global")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:
        """Resample ``(X, y)`` with the configured under-sampler.

        ``y`` is lifted out of ``X`` via ``target_column`` when not supplied
        separately; with no usable target the data passes through untouched, as
        it does for a ``method`` no builder recognizes.

        Raises:
            ValueError: If any feature column is non-numeric, which imblearn
                cannot resample. Encode first.
        """
        return apply_dual_engine(
            (X, y) if y is not None else X,
            params,
            {
                "polars": lambda Xi, yi, p: _resample_polars(
                    Xi, yi, p, _build_undersampler, "random_under_sampling"
                ),
                "pandas": lambda Xi, yi, p: _resample_pandas(
                    Xi, yi, p, _build_undersampler, "random_under_sampling"
                ),
            },
        )

apply(X, y, params)

Resample (X, y) with the configured under-sampler.

y is lifted out of X via target_column when not supplied separately; with no usable target the data passes through untouched, as it does for a method no builder recognizes.

Raises:

Type	Description
ValueError	If any feature column is non-numeric, which imblearn cannot resample. Encode first.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:
    """Resample ``(X, y)`` with the configured under-sampler.

    ``y`` is lifted out of ``X`` via ``target_column`` when not supplied
    separately; with no usable target the data passes through untouched, as
    it does for a ``method`` no builder recognizes.

    Raises:
        ValueError: If any feature column is non-numeric, which imblearn
            cannot resample. Encode first.
    """
    return apply_dual_engine(
        (X, y) if y is not None else X,
        params,
        {
            "polars": lambda Xi, yi, p: _resample_polars(
                Xi, yi, p, _build_undersampler, "random_under_sampling"
            ),
            "pandas": lambda Xi, yi, p: _resample_pandas(
                Xi, yi, p, _build_undersampler, "random_under_sampling"
            ),
        },
    )

inference_capability(state, *, engine) staticmethod

Require full class context; prediction still skips every sampling method.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Require full class context; prediction still skips every sampling method."""
    if engine not in ("pandas", "polars"):
        return None
    UndersamplingApplier.validate_inference_state(state)
    if not _under_context_known(state):
        return None
    return ExecutionCapability(engine, "apply", "local", "filter", "global")

validate_inference_state(raw) staticmethod

Inspect saved selection settings without loading the optional sampler package.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved selection settings without loading the optional sampler package."""
    return _validate_resampling_state(raw, "undersampling", "random_under_sampling")

UndersamplingCalculator

Bases: BaseCalculator

Configure class balancing by undersampling the majority class.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@NodeRegistry.register("Undersampling", UndersamplingApplier)
@node_meta(
    id="Undersampling",
    name="Undersampling",
    category="Preprocessing",
    description="Resample dataset to balance classes by undersampling majority class.",
    params={
        "method": "random_under_sampling",
        "target_column": "target",
        "sampling_strategy": "auto",
    },
    learns_from_data=True,
)
class UndersamplingCalculator(BaseCalculator):
    """Configure class balancing by undersampling the majority class."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Resampling changes row counts only; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> UndersamplingArtifact:  # pylint: disable=arguments-differ
        """Snapshot the sampler settings from ``config`` into the artifact.

        Neither ``_X`` nor ``_y`` is read: the artifact is pure configuration.
        Several of its keys are sampler-specific, so each method consumes only
        the subset its imblearn class accepts.
        """
        return {
            "type": "undersampling",
            "method": config.get("method", "random_under_sampling"),
            "target_column": config.get("target_column"),
            "sampling_strategy": config.get("sampling_strategy", "auto"),
            "random_state": config.get("random_state", 42),
            "replacement": config.get("replacement", False),
            "version": config.get("version", 1),  # For NearMiss
            "n_neighbors": config.get("n_neighbors", 3),  # For EditedNearestNeighbours
            "kind_sel": config.get("kind_sel", "all"),  # For EditedNearestNeighbours
            "n_jobs": config.get("n_jobs", -1),
        }

fit(_X, _y, config)

Snapshot the sampler settings from config into the artifact.

Neither _X nor _y is read: the artifact is pure configuration. Several of its keys are sampler-specific, so each method consumes only the subset its imblearn class accepts.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

@fit_method
def fit(self, _X: Any, _y: Any, config: dict[str, Any]) -> UndersamplingArtifact:  # pylint: disable=arguments-differ
    """Snapshot the sampler settings from ``config`` into the artifact.

    Neither ``_X`` nor ``_y`` is read: the artifact is pure configuration.
    Several of its keys are sampler-specific, so each method consumes only
    the subset its imblearn class accepts.
    """
    return {
        "type": "undersampling",
        "method": config.get("method", "random_under_sampling"),
        "target_column": config.get("target_column"),
        "sampling_strategy": config.get("sampling_strategy", "auto"),
        "random_state": config.get("random_state", 42),
        "replacement": config.get("replacement", False),
        "version": config.get("version", 1),  # For NearMiss
        "n_neighbors": config.get("n_neighbors", 3),  # For EditedNearestNeighbours
        "kind_sel": config.get("kind_sel", "all"),  # For EditedNearestNeighbours
        "n_jobs": config.get("n_jobs", -1),
    }

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/resampling.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Resampling changes row counts only; column set is preserved.
    return input_schema

UnivariateSelectionApplier

Bases: BaseApplier

Remove the columns a univariate statistical test rejected.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

class UnivariateSelectionApplier(BaseApplier):
    """Remove the columns a univariate statistical test rejected."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved selection membership while preserving both fitted schemas.

        A no-target fit uses ``scores``/``pvalues`` reporting fields; a scored
        fit uses ``feature_scores``/``p_values``. Reports and the fitted method
        are unused by apply and remain untouched, including nonfinite scores.
        """
        reports = (
            {"scores", "pvalues"}
            if type(raw) is dict and "scores" in raw
            else {"feature_scores", "p_values"}
        )
        fields = {"type", "selected_columns", "candidate_columns", "method", "drop_columns"}
        if not local_state_fields(raw, "univariate_selection", fields | reports, allow_empty=True):
            return raw
        selected = _local_selection_columns(raw["selected_columns"])
        candidates = _local_selection_columns(raw["candidate_columns"])
        if not set(selected).issubset(candidates):
            raise ValueError("Fitted selected columns must belong to candidate columns.")
        _local_selection_drop_flag(raw["drop_columns"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe existing local saved apply without admitting partition workers."""
        if engine not in {"pandas", "polars"}:
            return None
        UnivariateSelectionApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Drop ``candidate_columns - selected_columns`` from ``X``.

        Honours the artifact's ``drop_columns`` flag; the recorded
        ``feature_scores`` and ``p_values`` are reporting metadata only.
        """
        return apply_dual_engine(
            X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
        )

apply(X, _y, params)

Drop candidate_columns - selected_columns from X.

Honours the artifact's drop_columns flag; the recorded feature_scores and p_values are reporting metadata only.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Drop ``candidate_columns - selected_columns`` from ``X``.

    Honours the artifact's ``drop_columns`` flag; the recorded
    ``feature_scores`` and ``p_values`` are reporting metadata only.
    """
    return apply_dual_engine(
        X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe existing local saved apply without admitting partition workers.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe existing local saved apply without admitting partition workers."""
    if engine not in {"pandas", "polars"}:
        return None
    UnivariateSelectionApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved selection membership while preserving both fitted schemas.

A no-target fit uses scores/pvalues reporting fields; a scored fit uses feature_scores/p_values. Reports and the fitted method are unused by apply and remain untouched, including nonfinite scores.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved selection membership while preserving both fitted schemas.

    A no-target fit uses ``scores``/``pvalues`` reporting fields; a scored
    fit uses ``feature_scores``/``p_values``. Reports and the fitted method
    are unused by apply and remain untouched, including nonfinite scores.
    """
    reports = (
        {"scores", "pvalues"}
        if type(raw) is dict and "scores" in raw
        else {"feature_scores", "p_values"}
    )
    fields = {"type", "selected_columns", "candidate_columns", "method", "drop_columns"}
    if not local_state_fields(raw, "univariate_selection", fields | reports, allow_empty=True):
        return raw
    selected = _local_selection_columns(raw["selected_columns"])
    candidates = _local_selection_columns(raw["candidate_columns"])
    if not set(selected).issubset(candidates):
        raise ValueError("Fitted selected columns must belong to candidate columns.")
    _local_selection_drop_flag(raw["drop_columns"])
    return raw

UnivariateSelectionCalculator

Bases: BaseCalculator

Fit a univariate statistical selector over numeric candidate columns.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

@NodeRegistry.register("UnivariateSelection", UnivariateSelectionApplier)
@node_meta(
    id="UnivariateSelection",
    name="Univariate Selection",
    category="Feature Selection",
    description="Select best features based on univariate statistical tests.",
    params={"method": "SelectKBest", "score_func": "Auto", "k": 10},
    learns_from_data=True,
)
class UnivariateSelectionCalculator(BaseCalculator):
    """Fit a univariate statistical selector over numeric candidate columns."""

    @fit_method
    def fit(self, X: Any, y: Any, config: dict[str, Any]) -> UnivariateSelectionArtifact:  # pylint: disable=arguments-differ
        """Return the features the score function's support keeps.

        A missing target is normally an error, but ``allow_missing_target``
        downgrades it to a keep-everything artifact so unsupervised pipelines
        can still run this node. The scoring inputs are adjusted first — missing
        values become 0, and chi2 features are MinMax-rescaled when any value is
        negative — because both shift the resulting scores.
        """
        target_col = config.get("target_column")
        # Resolve candidates natively on the raw frame first (Polars-safe),
        # then convert only the columns actually needed.
        cols = _resolve_candidate_columns(X, config, target_col)
        if not cols:
            return cast(UnivariateSelectionArtifact, {})

        needed_cols = cols if y is not None or not target_col else [*cols, target_col]
        X_pd = select_then_to_pandas(X, needed_cols)

        y = _extract_target(X_pd, y, target_col)
        if y is None and not config.get("allow_missing_target", False):
            logger.error(
                f"UnivariateSelection requires target column '{target_col}' "
                "to be present in training data."
            )
            return cast(UnivariateSelectionArtifact, {})

        method = config.get("method", "select_k_best")
        score_func_name = config.get("score_func")
        problem_type = _resolve_problem_type(config.get("problem_type", "auto"), y)
        score_func = _resolve_score_function(score_func_name, problem_type)

        selector = _build_univariate_selector(method, score_func, config)
        if selector is None:
            return cast(UnivariateSelectionArtifact, {})

        X_np, _ = SklearnBridge.to_sklearn(_fillna_zero_with_warning(X_pd, cols))
        X_np = _maybe_chi2_rescale(X_np, score_func_name)

        if y is None:
            return _univariate_no_target_artifact(cols, method, config)

        selector.fit(X_np, _univariate_target(y, problem_type))
        support = selector.get_support()
        selected_cols = [c for c, s in zip(cols, support, strict=True) if s]
        scores, pvalues = _univariate_score_dicts(selector, cols)

        return cast(
            UnivariateSelectionArtifact,
            {
                "type": "univariate_selection",
                "selected_columns": selected_cols,
                "candidate_columns": cols,
                "method": method,
                "drop_columns": config.get("drop_columns", True),
                "feature_scores": scores,
                "p_values": pvalues,
            },
        )

fit(X, y, config)

Return the features the score function's support keeps.

A missing target is normally an error, but allow_missing_target downgrades it to a keep-everything artifact so unsupervised pipelines can still run this node. The scoring inputs are adjusted first — missing values become 0, and chi2 features are MinMax-rescaled when any value is negative — because both shift the resulting scores.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/univariate.py

@fit_method
def fit(self, X: Any, y: Any, config: dict[str, Any]) -> UnivariateSelectionArtifact:  # pylint: disable=arguments-differ
    """Return the features the score function's support keeps.

    A missing target is normally an error, but ``allow_missing_target``
    downgrades it to a keep-everything artifact so unsupervised pipelines
    can still run this node. The scoring inputs are adjusted first — missing
    values become 0, and chi2 features are MinMax-rescaled when any value is
    negative — because both shift the resulting scores.
    """
    target_col = config.get("target_column")
    # Resolve candidates natively on the raw frame first (Polars-safe),
    # then convert only the columns actually needed.
    cols = _resolve_candidate_columns(X, config, target_col)
    if not cols:
        return cast(UnivariateSelectionArtifact, {})

    needed_cols = cols if y is not None or not target_col else [*cols, target_col]
    X_pd = select_then_to_pandas(X, needed_cols)

    y = _extract_target(X_pd, y, target_col)
    if y is None and not config.get("allow_missing_target", False):
        logger.error(
            f"UnivariateSelection requires target column '{target_col}' "
            "to be present in training data."
        )
        return cast(UnivariateSelectionArtifact, {})

    method = config.get("method", "select_k_best")
    score_func_name = config.get("score_func")
    problem_type = _resolve_problem_type(config.get("problem_type", "auto"), y)
    score_func = _resolve_score_function(score_func_name, problem_type)

    selector = _build_univariate_selector(method, score_func, config)
    if selector is None:
        return cast(UnivariateSelectionArtifact, {})

    X_np, _ = SklearnBridge.to_sklearn(_fillna_zero_with_warning(X_pd, cols))
    X_np = _maybe_chi2_rescale(X_np, score_func_name)

    if y is None:
        return _univariate_no_target_artifact(cols, method, config)

    selector.fit(X_np, _univariate_target(y, problem_type))
    support = selector.get_support()
    selected_cols = [c for c, s in zip(cols, support, strict=True) if s]
    scores, pvalues = _univariate_score_dicts(selector, cols)

    return cast(
        UnivariateSelectionArtifact,
        {
            "type": "univariate_selection",
            "selected_columns": selected_cols,
            "candidate_columns": cols,
            "method": method,
            "drop_columns": config.get("drop_columns", True),
            "feature_scores": scores,
            "p_values": pvalues,
        },
    )

ValueReplacementApplier

Bases: BaseApplier

Replace configured values in the resolved columns, leaving other cells alone.

The pandas and polars paths must agree value-for-value. A mapping wins over to_replace/value when both are configured, and may be flat (one map applied to every column) or nested ({column: {old: new}}). Mapping and to_replace keys are coerced to the column's dtype before lookup. A list of keys can share one replacement or have a same-length list of replacements. A column selection resolving to nothing is a no-op.

Numeric rules on integer columns must fit the existing integer dtype; fractional, infinite and out-of-range replacements require an explicit Casting step. Missing replacements use native nullable integers. These checks use configured rules before matching rows, including empty requests. Nonnumeric replacements retain their native engine behavior.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

class ValueReplacementApplier(BaseApplier):
    """Replace configured values in the resolved columns, leaving other cells alone.

    The pandas and polars paths must agree value-for-value. A ``mapping`` wins
    over ``to_replace``/``value`` when both are configured, and may be flat
    (one map applied to every column) or nested (``{column: {old: new}}``).
    Mapping and ``to_replace`` keys are coerced to the column's dtype before
    lookup. A list of keys can share one replacement or have a same-length
    list of replacements. A column selection resolving to nothing is a no-op.

    Numeric rules on integer columns must fit the existing integer dtype;
    fractional, infinite and out-of-range replacements require an explicit
    Casting step. Missing replacements use native nullable integers. These
    checks use configured rules before matching rows, including empty requests.
    Nonnumeric replacements retain their native engine behavior.
    """

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect scalar, list, tuple and dictionary state without coercing saved rules."""
        local_state_fields(
            raw, "value_replacement", {"type", "columns", "mapping", "to_replace", "value"}
        )
        _columns(raw["columns"])
        _validate_replacement_rules(raw)
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe row dependency without claiming chunk-invariant dtype inference."""
        if engine not in ("pandas", "polars"):
            return None
        ValueReplacementApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Dispatch the replacement to the pandas or polars path; ``y`` passes through."""
        return apply_dual_engine(
            X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        valid = resolve_valid_columns(X, params.get("columns", []))
        if not valid:
            return X, _y
        exprs = _value_replacement_exprs_polars(
            X,
            valid,
            params.get("mapping"),
            params.get("to_replace"),
            params.get("value"),
        )
        return (X.with_columns(exprs) if exprs else X), _y

    @staticmethod
    def _apply_pandas(X: Any, _y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        valid = resolve_valid_columns(X, params.get("columns", []))
        if not valid:
            return X, _y
        df_out = X.copy()
        df_out = _apply_value_replacement_pandas(
            df_out,
            valid,
            params.get("mapping"),
            params.get("to_replace"),
            params.get("value"),
        )
        return df_out, _y

apply(X, _y, params)

Dispatch the replacement to the pandas or polars path; y passes through.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Dispatch the replacement to the pandas or polars path; ``y`` passes through."""
    return apply_dual_engine(
        X, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe row dependency without claiming chunk-invariant dtype inference.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe row dependency without claiming chunk-invariant dtype inference."""
    if engine not in ("pandas", "polars"):
        return None
    ValueReplacementApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect scalar, list, tuple and dictionary state without coercing saved rules.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect scalar, list, tuple and dictionary state without coercing saved rules."""
    local_state_fields(
        raw, "value_replacement", {"type", "columns", "mapping", "to_replace", "value"}
    )
    _columns(raw["columns"])
    _validate_replacement_rules(raw)
    return raw

ValueReplacementCalculator

Bases: BaseCalculator

Resolve value-replacement config into an artifact; nothing is learned from data.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

@NodeRegistry.register("ValueReplacement", ValueReplacementApplier)
@node_meta(
    id="ValueReplacement",
    name="Replace Values",
    category="Cleaning",
    description="Replace specified values with new values.",
    params={"columns": [], "mapping": {}},
    learns_from_data=False,
)
class ValueReplacementCalculator(BaseCalculator):
    """Resolve value-replacement config into an artifact; nothing is learned from data."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Preserve columns and report only output dtypes guaranteed by the active rules.

        Integer null rules expose their native nullable type. Data-dependent or
        unsupported replacements omit the affected dtype; consumers can still
        validate column references and display the type as unknown.
        """
        dtypes = dict(input_schema.dtypes)
        for column in config.get("columns") or []:
            if column not in dtypes:
                continue
            try:
                label = _replacement_output_dtype(
                    dtypes[column], _schema_replacement_rules(config, column)
                )
            except (TypeError, ValueError, OverflowError):
                label = None
            if label is None:
                dtypes.pop(column)
            else:
                dtypes[column] = label
        if dtypes == input_schema.dtypes:
            return input_schema
        return SkyulfSchema.from_columns(input_schema.columns, dtypes)

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ValueReplacementArtifact:  # pylint: disable=arguments-differ
        """Build the artifact, folding the UI's ``replacements`` pairs into a flat mapping.

        A non-empty ``replacements`` list overrides any configured ``mapping``.
        ``to_replace``/``value`` are carried through untouched for the applier
        to fall back on when no mapping survived.
        """
        cols = resolve_columns(X, config)
        return {
            "type": "value_replacement",
            "columns": cols,
            "mapping": _configured_mapping(config),
            "to_replace": config.get("to_replace"),
            "value": config.get("value"),
        }

fit(X, _y, config)

Build the artifact, folding the UI's replacements pairs into a flat mapping.

A non-empty replacements list overrides any configured mapping. to_replace/value are carried through untouched for the applier to fall back on when no mapping survived.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ValueReplacementArtifact:  # pylint: disable=arguments-differ
    """Build the artifact, folding the UI's ``replacements`` pairs into a flat mapping.

    A non-empty ``replacements`` list overrides any configured ``mapping``.
    ``to_replace``/``value`` are carried through untouched for the applier
    to fall back on when no mapping survived.
    """
    cols = resolve_columns(X, config)
    return {
        "type": "value_replacement",
        "columns": cols,
        "mapping": _configured_mapping(config),
        "to_replace": config.get("to_replace"),
        "value": config.get("value"),
    }

infer_output_schema(input_schema, config)

Preserve columns and report only output dtypes guaranteed by the active rules.

Integer null rules expose their native nullable type. Data-dependent or unsupported replacements omit the affected dtype; consumers can still validate column references and display the type as unknown.

Source code in skyulf-core/skyulf/preprocessing/cleaning/value_replacement.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Preserve columns and report only output dtypes guaranteed by the active rules.

    Integer null rules expose their native nullable type. Data-dependent or
    unsupported replacements omit the affected dtype; consumers can still
    validate column references and display the type as unknown.
    """
    dtypes = dict(input_schema.dtypes)
    for column in config.get("columns") or []:
        if column not in dtypes:
            continue
        try:
            label = _replacement_output_dtype(
                dtypes[column], _schema_replacement_rules(config, column)
            )
        except (TypeError, ValueError, OverflowError):
            label = None
        if label is None:
            dtypes.pop(column)
        else:
            dtypes[column] = label
    if dtypes == input_schema.dtypes:
        return input_schema
    return SkyulfSchema.from_columns(input_schema.columns, dtypes)

VarianceThresholdApplier

Bases: BaseApplier

Remove the columns a fitted variance threshold rejected.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

class VarianceThresholdApplier(BaseApplier):
    """Remove the columns a fitted variance threshold rejected."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved selection membership without recomputing variances.

        Empty fitted artifacts and selections are valid. The fit-only threshold
        and variances remain untouched, including undefined all-missing variance.
        """
        fields = {
            "type",
            "selected_columns",
            "candidate_columns",
            "threshold",
            "drop_columns",
            "variances",
        }
        if not local_state_fields(raw, "variance_threshold", fields, allow_empty=True):
            return raw
        selected = _local_selection_columns(raw["selected_columns"])
        candidates = _local_selection_columns(raw["candidate_columns"])
        if not set(selected).issubset(candidates):
            raise ValueError("Fitted selected columns must belong to candidate columns.")
        _local_selection_drop_flag(raw["drop_columns"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe existing local saved apply without admitting partition workers."""
        if engine not in {"pandas", "polars"}:
            return None
        VarianceThresholdApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Drop ``candidate_columns - selected_columns`` from ``X``.

        Honours the artifact's ``drop_columns`` flag; when it is false the frame
        passes through untouched.
        """
        return apply_dual_engine(
            X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
        )

apply(X, _y, params)

Drop candidate_columns - selected_columns from X.

Honours the artifact's drop_columns flag; when it is false the frame passes through untouched.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

@apply_method
def apply(self, X: Any, _y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Drop ``candidate_columns - selected_columns`` from ``X``.

    Honours the artifact's ``drop_columns`` flag; when it is false the frame
    passes through untouched.
    """
    return apply_dual_engine(
        X, params, {"polars": _drop_selected_polars, "pandas": _drop_selected_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe existing local saved apply without admitting partition workers.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe existing local saved apply without admitting partition workers."""
    if engine not in {"pandas", "polars"}:
        return None
    VarianceThresholdApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect saved selection membership without recomputing variances.

Empty fitted artifacts and selections are valid. The fit-only threshold and variances remain untouched, including undefined all-missing variance.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved selection membership without recomputing variances.

    Empty fitted artifacts and selections are valid. The fit-only threshold
    and variances remain untouched, including undefined all-missing variance.
    """
    fields = {
        "type",
        "selected_columns",
        "candidate_columns",
        "threshold",
        "drop_columns",
        "variances",
    }
    if not local_state_fields(raw, "variance_threshold", fields, allow_empty=True):
        return raw
    selected = _local_selection_columns(raw["selected_columns"])
    candidates = _local_selection_columns(raw["candidate_columns"])
    if not set(selected).issubset(candidates):
        raise ValueError("Fitted selected columns must belong to candidate columns.")
    _local_selection_drop_flag(raw["drop_columns"])
    return raw

VarianceThresholdCalculator

Bases: BaseCalculator

Fit sklearn.feature_selection.VarianceThreshold on numeric columns.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

@NodeRegistry.register("VarianceThreshold", VarianceThresholdApplier)
@node_meta(
    id="VarianceThreshold",
    name="Variance Threshold",
    category="Feature Selection",
    description="Remove features with low variance.",
    params={"threshold": 0.0},
    learns_from_data=True,
)
class VarianceThresholdCalculator(BaseCalculator):
    """Fit ``sklearn.feature_selection.VarianceThreshold`` on numeric columns."""

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> VarianceThresholdArtifact:  # pylint: disable=arguments-differ
        """Record which numeric columns clear ``config["threshold"]`` variance.

        Binary and constant columns are kept as candidates rather than excluded
        up front, so the threshold itself decides their fate. Returns an empty
        artifact — a no-op passthrough — when no numeric column is resolved.
        If no candidate clears the threshold, records an empty selection so
        apply can remove all candidates while preserving other columns.
        """
        threshold = config.get("threshold", 0.0)
        drop_columns = config.get("drop_columns", True)

        X_np, cols = resolve_columns_then_to_numpy(
            X,
            config,
            lambda d: detect_numeric_columns(d, exclude_binary=False, exclude_constant=False),
        )
        if not cols:
            return cast(VarianceThresholdArtifact, {})

        selector = VarianceThreshold(threshold=threshold)
        try:
            selector.fit(X_np)
        except ValueError as exc:
            # sklearn has computed variances but refuses an empty selection.
            if not str(exc).startswith("No feature in X meets the variance threshold"):
                raise
            all_missing = np.isnan(np.asarray(X_np, dtype=np.float64)).all(axis=0)
            if np.any(~np.isfinite(selector.variances_) & ~all_missing):
                # Only all-missing columns may have an undefined variance.
                raise
            selected_cols = []
        else:
            support = selector.get_support()
            selected_cols = [c for c, s in zip(cols, support, strict=True) if s]
        variances = (
            dict(zip(cols, selector.variances_.tolist(), strict=True))
            if hasattr(selector, "variances_")
            else {}
        )
        return cast(
            VarianceThresholdArtifact,
            {
                "type": "variance_threshold",
                "selected_columns": selected_cols,
                "candidate_columns": cols,
                "threshold": threshold,
                "drop_columns": drop_columns,
                "variances": variances,
            },
        )

fit(X, _y, config)

Record which numeric columns clear config["threshold"] variance.

Binary and constant columns are kept as candidates rather than excluded up front, so the threshold itself decides their fate. Returns an empty artifact — a no-op passthrough — when no numeric column is resolved. If no candidate clears the threshold, records an empty selection so apply can remove all candidates while preserving other columns.

Source code in skyulf-core/skyulf/preprocessing/feature_selection/variance.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> VarianceThresholdArtifact:  # pylint: disable=arguments-differ
    """Record which numeric columns clear ``config["threshold"]`` variance.

    Binary and constant columns are kept as candidates rather than excluded
    up front, so the threshold itself decides their fate. Returns an empty
    artifact — a no-op passthrough — when no numeric column is resolved.
    If no candidate clears the threshold, records an empty selection so
    apply can remove all candidates while preserving other columns.
    """
    threshold = config.get("threshold", 0.0)
    drop_columns = config.get("drop_columns", True)

    X_np, cols = resolve_columns_then_to_numpy(
        X,
        config,
        lambda d: detect_numeric_columns(d, exclude_binary=False, exclude_constant=False),
    )
    if not cols:
        return cast(VarianceThresholdArtifact, {})

    selector = VarianceThreshold(threshold=threshold)
    try:
        selector.fit(X_np)
    except ValueError as exc:
        # sklearn has computed variances but refuses an empty selection.
        if not str(exc).startswith("No feature in X meets the variance threshold"):
            raise
        all_missing = np.isnan(np.asarray(X_np, dtype=np.float64)).all(axis=0)
        if np.any(~np.isfinite(selector.variances_) & ~all_missing):
            # Only all-missing columns may have an undefined variance.
            raise
        selected_cols = []
    else:
        support = selector.get_support()
        selected_cols = [c for c, s in zip(cols, support, strict=True) if s]
    variances = (
        dict(zip(cols, selector.variances_.tolist(), strict=True))
        if hasattr(selector, "variances_")
        else {}
    )
    return cast(
        VarianceThresholdArtifact,
        {
            "type": "variance_threshold",
            "selected_columns": selected_cols,
            "candidate_columns": cols,
            "threshold": threshold,
            "drop_columns": drop_columns,
            "variances": variances,
        },
    )

WinsorizeApplier

Bases: BaseApplier

Clip values to the fitted percentile bounds; rows are never removed.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

class WinsorizeApplier(BaseApplier):
    """Clip values to the fitted percentile bounds; rows are never removed."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect learned bounds and metadata without recomputing request quantiles."""
        if not local_state_fields(
            raw,
            "winsorize",
            {"type", "bounds", "lower_percentile", "upper_percentile", "warnings"},
            allow_empty=True,
        ):
            return raw
        validate_fitted_bounds(raw["bounds"], partial=False)
        for field in ("lower_percentile", "upper_percentile"):
            value = raw[field]
            if isinstance(value, bool) or not isinstance(value, Real) or not 0 <= value <= 100:
                raise ValueError("Fitted percentiles must be numbers between zero and 100.")
        if type(raw["warnings"]) is not list or any(
            type(item) is not str for item in raw["warnings"]
        ):
            raise ValueError("Fitted warnings must be a list of strings.")
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe local row clipping with saved limits without admitting Spark workers."""
        if engine not in ("pandas", "polars"):
            return None
        WinsorizeApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Clip ``X`` to the fitted per-column bounds; ``y`` passes through untouched."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y. Winsorize never
        # filters rows, but the wrap keeps behavior consistent with the other
        # outlier nodes and avoids losing y from the returned tuple.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        numeric_cols = set(auto_detect_numeric_columns(X))
        exprs = []
        for col, bound in bounds.items():
            if col not in X.columns or col not in numeric_cols:
                continue
            exprs.append(_clip_polars_column(X, col, bound))
        return X.with_columns(exprs), y

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        bounds = params.get("bounds", {})
        if not bounds:
            return X, y

        df_out = X.copy()
        for col, bound in bounds.items():
            if col not in df_out.columns:
                continue
            if pd.api.types.is_numeric_dtype(df_out[col]) or is_decimal_series(df_out[col]):
                df_out[col] = _clip_pandas_column(df_out[col], bound, col)
        return df_out, y

apply(X, y, params)

Clip X to the fitted per-column bounds; y passes through untouched.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Clip ``X`` to the fitted per-column bounds; ``y`` passes through untouched."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y. Winsorize never
    # filters rows, but the wrap keeps behavior consistent with the other
    # outlier nodes and avoids losing y from the returned tuple.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe local row clipping with saved limits without admitting Spark workers.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe local row clipping with saved limits without admitting Spark workers."""
    if engine not in ("pandas", "polars"):
        return None
    WinsorizeApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Inspect learned bounds and metadata without recomputing request quantiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect learned bounds and metadata without recomputing request quantiles."""
    if not local_state_fields(
        raw,
        "winsorize",
        {"type", "bounds", "lower_percentile", "upper_percentile", "warnings"},
        allow_empty=True,
    ):
        return raw
    validate_fitted_bounds(raw["bounds"], partial=False)
    for field in ("lower_percentile", "upper_percentile"):
        value = raw[field]
        if isinstance(value, bool) or not isinstance(value, Real) or not 0 <= value <= 100:
            raise ValueError("Fitted percentiles must be numbers between zero and 100.")
    if type(raw["warnings"]) is not list or any(
        type(item) is not str for item in raw["warnings"]
    ):
        raise ValueError("Fitted warnings must be a list of strings.")
    return raw

WinsorizeCalculator

Bases: BaseCalculator

Fit per-column clip bounds at the configured lower/upper percentiles.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

@NodeRegistry.register("Winsorize", WinsorizeApplier)
@node_meta(
    id="Winsorize",
    name="Winsorization",
    category="Preprocessing",
    description="Limit extreme values in the data.",
    params={"lower_percentile": 5.0, "upper_percentile": 95.0, "columns": []},
    learns_from_data=True,
)
class WinsorizeCalculator(BaseCalculator):
    """Fit per-column clip bounds at the configured lower/upper percentiles."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Preserve integer columns and retain existing promotion for other selected types."""
        if not isinstance(input_schema, SkyulfSchema):
            return input_schema
        output = promote_configured_columns_to_float64(input_schema, config)
        for column, dtype in input_schema.dtypes.items():
            if dtype.lower().startswith(("int", "uint")):
                output = output.with_dtype(column, dtype)
        return output

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> WinsorizeArtifact:  # pylint: disable=arguments-differ
        """Compute each column's lower/upper quantiles; warn on empty or non-numeric ones."""
        if user_picked_no_columns(config):
            return {}

        lower_p = config.get("lower_percentile", 5.0)
        upper_p = config.get("upper_percentile", 95.0)
        X = X.to_native() if hasattr(X, "to_native") else X
        cols = resolve_columns(X, config, detect_numeric_columns)
        if not cols:
            return {}

        bounds: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series, integer = _fit_series(X, col)
            if series.empty:
                warnings.append(f"Column '{col}': Empty or non-numeric")
                continue
            bounds[col] = {
                "lower": _fit_bound(series, lower_p, col, integer=integer),
                "upper": _fit_bound(series, upper_p, col, integer=integer),
            }
            if integer:
                bounds[col] = _pandas_integer_bounds(series, bounds[col], col)

        return {
            "type": "winsorize",
            "bounds": bounds,
            "lower_percentile": lower_p,
            "upper_percentile": upper_p,
            "warnings": warnings,
        }

fit(X, _y, config)

Compute each column's lower/upper quantiles; warn on empty or non-numeric ones.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> WinsorizeArtifact:  # pylint: disable=arguments-differ
    """Compute each column's lower/upper quantiles; warn on empty or non-numeric ones."""
    if user_picked_no_columns(config):
        return {}

    lower_p = config.get("lower_percentile", 5.0)
    upper_p = config.get("upper_percentile", 95.0)
    X = X.to_native() if hasattr(X, "to_native") else X
    cols = resolve_columns(X, config, detect_numeric_columns)
    if not cols:
        return {}

    bounds: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series, integer = _fit_series(X, col)
        if series.empty:
            warnings.append(f"Column '{col}': Empty or non-numeric")
            continue
        bounds[col] = {
            "lower": _fit_bound(series, lower_p, col, integer=integer),
            "upper": _fit_bound(series, upper_p, col, integer=integer),
        }
        if integer:
            bounds[col] = _pandas_integer_bounds(series, bounds[col], col)

    return {
        "type": "winsorize",
        "bounds": bounds,
        "lower_percentile": lower_p,
        "upper_percentile": upper_p,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Preserve integer columns and retain existing promotion for other selected types.

Source code in skyulf-core/skyulf/preprocessing/outliers/winsorize.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Preserve integer columns and retain existing promotion for other selected types."""
    if not isinstance(input_schema, SkyulfSchema):
        return input_schema
    output = promote_configured_columns_to_float64(input_schema, config)
    for column, dtype in input_schema.dtypes.items():
        if dtype.lower().startswith(("int", "uint")):
            output = output.with_dtype(column, dtype)
    return output

ZScoreApplier

Bases: BaseApplier

Drop rows whose |z| against the fitted mean/std exceeds threshold in any column.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

class ZScoreApplier(BaseApplier):
    """Drop rows whose |z| against the fitted mean/std exceeds ``threshold`` in any column."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Inspect saved numeric statistics and native scalar threshold semantics."""
        if local_state_fields(
            raw, "zscore", {"type", "stats", "threshold", "warnings"}, allow_empty=True
        ):
            _validate_zscore_statistics(raw["stats"])
            if not isinstance(raw["threshold"], (Real, Decimal)):
                local_scalar(raw["threshold"], "ZScore threshold")
            validate_detector_warnings(raw["warnings"])
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe saved row filtering without bypassing the prediction row-count guard."""
        if engine not in ("pandas", "polars"):
            return None
        ZScoreApplier.validate_inference_state(state)
        effect = "filter" if state.get("stats") else "preserve"
        return ExecutionCapability(engine, "apply", "local", effect, "row")

    @apply_method
    def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
        """Filter rows through the fitted z-score threshold on the active engine."""
        # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
        # own unpack_pipeline_input doesn't silently drop y (leaving it
        # unfiltered when X rows are removed). Omit the wrap when y is None
        # to avoid apply_dual_engine's tuple-with-no-y warning log.
        input_data = (X, y) if y is not None else X
        return apply_dual_engine(
            input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
        )

    @staticmethod
    def _apply_polars(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        stats = params.get("stats", {})
        threshold = params.get("threshold", 3.0)
        if not stats:
            return X, y

        mask = pl.repeat(True, pl.len())
        for col, stat in stats.items():
            if col not in X.columns or stat["std"] == 0:
                continue
            values = pl.col(col).cast(pl.Float64, strict=False)
            z = (values - stat["mean"]) / stat["std"]
            col_mask = z.abs() <= threshold
            missing = values.is_null() | values.is_nan()
            mask = mask & (col_mask | missing)

        mask_series = X.select(mask.alias("mask")).get_column("mask")
        return X.filter(mask_series), _filter_y_polars(y, mask_series)

    @staticmethod
    def _apply_pandas(X: Any, y: Any, params: dict[str, Any]) -> tuple[Any, Any]:
        stats = params.get("stats", {})
        threshold = params.get("threshold", 3.0)
        if not stats:
            return X, y

        mask = pd.Series(True, index=X.index)
        for col, stat in stats.items():
            if col not in X.columns or stat["std"] == 0:
                continue
            series = pd.to_numeric(X[col], errors="coerce")
            z = (series - stat["mean"]) / stat["std"]
            col_mask = z.abs() <= threshold
            mask = mask & (col_mask | series.isna())

        return _apply_pandas_mask(X, y, mask)

apply(X, y, params)

Filter rows through the fitted z-score threshold on the active engine.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

@apply_method
def apply(self, X: Any, y: Any, params: dict[str, Any]) -> Any:  # pylint: disable=arguments-differ
    """Filter rows through the fitted z-score threshold on the active engine."""
    # apply_method already unpacked (X, y); re-wrap so apply_dual_engine's
    # own unpack_pipeline_input doesn't silently drop y (leaving it
    # unfiltered when X rows are removed). Omit the wrap when y is None
    # to avoid apply_dual_engine's tuple-with-no-y warning log.
    input_data = (X, y) if y is not None else X
    return apply_dual_engine(
        input_data, params, {"polars": self._apply_polars, "pandas": self._apply_pandas}
    )

inference_capability(state, *, engine) staticmethod

Describe saved row filtering without bypassing the prediction row-count guard.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe saved row filtering without bypassing the prediction row-count guard."""
    if engine not in ("pandas", "polars"):
        return None
    ZScoreApplier.validate_inference_state(state)
    effect = "filter" if state.get("stats") else "preserve"
    return ExecutionCapability(engine, "apply", "local", effect, "row")

validate_inference_state(raw) staticmethod

Inspect saved numeric statistics and native scalar threshold semantics.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Inspect saved numeric statistics and native scalar threshold semantics."""
    if local_state_fields(
        raw, "zscore", {"type", "stats", "threshold", "warnings"}, allow_empty=True
    ):
        _validate_zscore_statistics(raw["stats"])
        if not isinstance(raw["threshold"], (Real, Decimal)):
            local_scalar(raw["threshold"], "ZScore threshold")
        validate_detector_warnings(raw["warnings"])
    return raw

ZScoreCalculator

Bases: BaseCalculator

Fit per-column mean and population std (ddof=0) for z-score filtering.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

@NodeRegistry.register("ZScore", ZScoreApplier)
@node_meta(
    id="ZScore",
    name="Z-Score Outlier Removal",
    category="Preprocessing",
    description="Remove outliers using Z-Score.",
    params={"threshold": 3.0, "columns": []},
    learns_from_data=True,
)
class ZScoreCalculator(BaseCalculator):
    """Fit per-column mean and population std (``ddof=0``) for z-score filtering."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return the input schema unchanged: row filtering preserves the column set."""
        # Z-score removes outlier *rows*; column set is preserved.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ZScoreArtifact:  # pylint: disable=arguments-differ
        """Record each column's mean/std; warn on empty or zero-variance columns."""
        if user_picked_no_columns(config):
            return {}

        threshold = config.get("threshold", 3.0)
        # TODO(pandas-removal): mean/std alone would be numpy-safe (Polars std
        # ddof=0 already matches Pandas — verified), but this loop relies on
        # per-column pd.to_numeric(errors="coerce") to gracefully skip
        # non-numeric/mixed-dtype columns and .dropna() per column before
        # computing stats. Revisit once there's a native Polars equivalent
        # (pl.col(col).cast(pl.Float64, strict=False).drop_nulls()) wired in
        # to replace the coercion step without changing which columns get a
        # "non-numeric" warning.
        X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
        if not cols:
            return {}

        stats: dict[str, dict[str, float]] = {}
        warnings = []
        for col in cols:
            series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
            series = series[np.isfinite(series)]
            if series.empty:
                warnings.append(f"Column '{col}': Empty, non-numeric or non-finite")
                continue
            std = series.std(ddof=0)
            if std == 0:
                warnings.append(f"Column '{col}': Zero variance (std=0)")
                continue
            stats[col] = {"mean": series.mean(), "std": std}

        return {
            "type": "zscore",
            "stats": stats,
            "threshold": threshold,
            "warnings": warnings,
        }

fit(X, _y, config)

Record each column's mean/std; warn on empty or zero-variance columns.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> ZScoreArtifact:  # pylint: disable=arguments-differ
    """Record each column's mean/std; warn on empty or zero-variance columns."""
    if user_picked_no_columns(config):
        return {}

    threshold = config.get("threshold", 3.0)
    # TODO(pandas-removal): mean/std alone would be numpy-safe (Polars std
    # ddof=0 already matches Pandas — verified), but this loop relies on
    # per-column pd.to_numeric(errors="coerce") to gracefully skip
    # non-numeric/mixed-dtype columns and .dropna() per column before
    # computing stats. Revisit once there's a native Polars equivalent
    # (pl.col(col).cast(pl.Float64, strict=False).drop_nulls()) wired in
    # to replace the coercion step without changing which columns get a
    # "non-numeric" warning.
    X_pd, cols = resolve_columns_then_to_pandas(X, config, detect_numeric_columns)
    if not cols:
        return {}

    stats: dict[str, dict[str, float]] = {}
    warnings = []
    for col in cols:
        series = pd.to_numeric(X_pd[col], errors="coerce").dropna()
        series = series[np.isfinite(series)]
        if series.empty:
            warnings.append(f"Column '{col}': Empty, non-numeric or non-finite")
            continue
        std = series.std(ddof=0)
        if std == 0:
            warnings.append(f"Column '{col}': Zero variance (std=0)")
            continue
        stats[col] = {"mean": series.mean(), "std": std}

    return {
        "type": "zscore",
        "stats": stats,
        "threshold": threshold,
        "warnings": warnings,
    }

infer_output_schema(input_schema, config)

Return the input schema unchanged: row filtering preserves the column set.

Source code in skyulf-core/skyulf/preprocessing/outliers/zscore.py

def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return the input schema unchanged: row filtering preserves the column set."""
    # Z-score removes outlier *rows*; column set is preserved.
    return input_schema

column_step(name, function, *, output, replace=False, params=None, inference_context=None)

Build columns from function(df) or function(df, params).

inference_context optionally declares row, group, window or global input dependence for diagnostics. It does not authorize distributed execution.

Source code in skyulf-core/skyulf/preprocessing/function_steps.py

def column_step(
    name: str,
    function: Callable[..., Any],
    *,
    output: str | list[str],
    replace: bool = False,
    params: dict[str, Any] | None = None,
    inference_context: str | None = None,
) -> dict[str, Any]:
    """Build columns from ``function(df)`` or ``function(df, params)``.

    ``inference_context`` optionally declares row, group, window or global input
    dependence for diagnostics. It does not authorize distributed execution.
    """
    return {
        "name": name,
        "transformer": COLUMN_STEP,
        "params": {
            "function": function_ref(function),
            "output": _outputs(output),
            "replace": bool(replace),
            "params": _user_params(params),
            **_inference_context(inference_context),
        },
    }

filter_step(name, function, *, columns, params=None, inference_context=None)

Build a pre-split filter keeping rows where function(df) is True.

inference_context optionally declares row, group, window or global input dependence for diagnostics. Undeclared callback context remains unknown.

Source code in skyulf-core/skyulf/preprocessing/function_steps.py

def filter_step(
    name: str,
    function: Callable[..., Any],
    *,
    columns: list[str],
    params: dict[str, Any] | None = None,
    inference_context: str | None = None,
) -> dict[str, Any]:
    """Build a pre-split filter keeping rows where ``function(df)`` is True.

    ``inference_context`` optionally declares row, group, window or global input
    dependence for diagnostics. Undeclared callback context remains unknown.
    """
    if not isinstance(columns, list) or not columns or len(set(columns)) != len(columns):
        raise ValueError("filter columns must be a nonempty list of unique column names.")
    return {
        "name": name,
        "transformer": FILTER_STEP,
        "params": {
            "function": function_ref(function),
            "columns": list(columns),
            "params": _user_params(params),
            **_inference_context(inference_context),
        },
        "pre_split": {
            "effect": "filter",
            "required_columns": list(columns),
            "learns_from_data": False,
        },
    }

fitted_step(name, learn, apply, *, output, replace=False, params=None, inference_context=None)

Build a step reusing learn(df, y) state in apply(df, state).

inference_context optionally declares row, group, window or global input dependence for diagnostics. Learned state alone supplies no such promise.

Source code in skyulf-core/skyulf/preprocessing/function_steps.py

def fitted_step(
    name: str,
    learn: Callable[..., Any],
    apply: Callable[..., Any],
    *,
    output: str | list[str],
    replace: bool = False,
    params: dict[str, Any] | None = None,
    inference_context: str | None = None,
) -> dict[str, Any]:
    """Build a step reusing ``learn(df, y)`` state in ``apply(df, state)``.

    ``inference_context`` optionally declares row, group, window or global input
    dependence for diagnostics. Learned state alone supplies no such promise.
    """
    return {
        "name": name,
        "transformer": FITTED_STEP,
        "params": {
            "learn": function_ref(learn),
            "apply": function_ref(apply),
            "output": _outputs(output),
            "replace": bool(replace),
            "params": _user_params(params),
            **_inference_context(inference_context),
        },
    }

frame_rows(frame)

Row count for pandas/polars frames and numpy arrays (-1 if unknowable).

Source code in skyulf-core/skyulf/preprocessing/fold_adapter.py

def frame_rows(frame: Any) -> int:
    """Row count for pandas/polars frames and numpy arrays (-1 if unknowable)."""
    height = getattr(frame, "height", None)
    if isinstance(height, int):
        return height
    try:
        return len(frame)
    except TypeError:
        return -1

validate_schema(expected, actual, *, check_dtypes=False, check_order=False, where='input')

Validate a live DataFrame against an expected schema.

Thin convenience wrapper: builds a :class:SkyulfSchema from actual (Pandas/Polars/wrapper frame) and delegates to :meth:SkyulfSchema.assert_compatible. Raises :class:SchemaMismatchError on any violation.

Source code in skyulf-core/skyulf/core/schema.py

def validate_schema(
    expected: SkyulfSchema,
    actual: Any,
    *,
    check_dtypes: bool = False,
    check_order: bool = False,
    where: str = "input",
) -> None:
    """Validate a live DataFrame against an ``expected`` schema.

    Thin convenience wrapper: builds a :class:`SkyulfSchema` from ``actual``
    (Pandas/Polars/wrapper frame) and delegates to
    :meth:`SkyulfSchema.assert_compatible`. Raises
    :class:`SchemaMismatchError` on any violation.
    """
    actual_schema = (
        actual if isinstance(actual, SkyulfSchema) else SkyulfSchema.from_dataframe(actual)
    )
    expected.assert_compatible(
        actual_schema,
        check_dtypes=check_dtypes,
        check_order=check_order,
        where=where,
    )