fix(DONT MERGE): duckdb>=1.4.1 typing & warnings

narwhals/_duckdb/dataframe.py

@@ -101,7 +101,7 @@ def to_narwhals(
         if self._version is Version.V1:
             from narwhals.stable.v1 import DataFrame as DataFrameV1
 
-            return DataFrameV1(self, level="interchange")  # type: ignore[no-any-return]
+            return DataFrameV1(self, level="interchange")
         return self._version.lazyframe(self, level="lazy")
 
     def __narwhals_dataframe__(self) -> Self:  # pragma: no cover
@@ -115,7 +115,7 @@ def __narwhals_lazyframe__(self) -> Self:
         return self
 
     def __native_namespace__(self) -> ModuleType:
-        return get_duckdb()  # type: ignore[no-any-return]
+        return get_duckdb()
 
     def __narwhals_namespace__(self) -> DuckDBNamespace:
         from narwhals._duckdb.namespace import DuckDBNamespace
@@ -137,12 +137,8 @@ def collect(
         if backend is None or backend is Implementation.PYARROW:
             from narwhals._arrow.dataframe import ArrowDataFrame
 
-            if self._backend_version < (1, 4):
-                ret = self.native.arrow()
-            else:  # pragma: no cover
-                ret = self.native.fetch_arrow_table()
             return ArrowDataFrame(
-                ret,
+                self.native.fetch_arrow_table(),
                 validate_backend_version=True,
                 version=self._version,
                 validate_column_names=True,
@@ -259,7 +255,7 @@ def to_pandas(self) -> pd.DataFrame:
 
     def to_arrow(self) -> pa.Table:
         # only if version is v1, keep around for backcompat
-        return self.lazy().collect(Implementation.PYARROW).native  # type: ignore[no-any-return]
+        return self.lazy().collect(Implementation.PYARROW).native
 
     def _with_version(self, version: Version) -> Self:
         return self.__class__(self.native, version=version)

narwhals/_duckdb/namespace.py

@@ -37,6 +37,9 @@
     from narwhals._utils import Version
     from narwhals.typing import ConcatMethod, IntoDType, NonNestedLiteral
 
+BIGINT = duckdb_dtypes.BIGINT
+VARCHAR = duckdb_dtypes.VARCHAR
+
 
 class DuckDBNamespace(
     SQLNamespace[DuckDBLazyFrame, DuckDBExpr, "DuckDBPyRelation", Expression]
@@ -108,9 +111,9 @@ def func(df: DuckDBLazyFrame) -> list[Expression]:
                 cols_separated = [
                     y
                     for x in [
-                        (col.cast(duckdb_dtypes.VARCHAR),)
+                        (col.cast(VARCHAR),)
                         if i == len(cols) - 1
-                        else (col.cast(duckdb_dtypes.VARCHAR), lit(separator))
+                        else (col.cast(VARCHAR), lit(separator))
                         for i, col in enumerate(cols)
                     ]
                     for y in x
@@ -130,9 +133,7 @@ def func(cols: Iterable[Expression]) -> Expression:
             cols = list(cols)
             return reduce(
                 operator.add, (CoalesceOperator(col, lit(0)) for col in cols)
-            ) / reduce(
-                operator.add, (col.isnotnull().cast(duckdb_dtypes.BIGINT) for col in cols)
-            )
+            ) / reduce(operator.add, (col.isnotnull().cast(BIGINT) for col in cols))
 
         return self._expr._from_elementwise_horizontal_op(func, *exprs)
 

narwhals/_duckdb/series.py

@@ -24,7 +24,7 @@ def __narwhals_series__(self) -> Self:
         return self
 
     def __native_namespace__(self) -> ModuleType:
-        return get_duckdb()  # type: ignore[no-any-return]
+        return get_duckdb()
 
     @property
     def dtype(self) -> DType:

narwhals/_duckdb/typing.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, TypedDict
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Literal, Protocol, TypedDict, overload
 
-if TYPE_CHECKING:
-    from collections.abc import Sequence
+from narwhals._typing_compat import TypeVar
 
-    from duckdb import Expression
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection, Expression
+    from typing_extensions import TypeAlias, TypeIs
 
 
 class WindowExpressionKwargs(TypedDict, total=False):
@@ -16,3 +18,101 @@ class WindowExpressionKwargs(TypedDict, total=False):
     descending: Sequence[bool]
     nulls_last: Sequence[bool]
     ignore_nulls: bool
+
+
+_Children: TypeAlias = Sequence[tuple[str, Any]]
+T_co = TypeVar("T_co", covariant=True, bound=_Children, default=_Children)
+DTypeT_co = TypeVar("DTypeT_co", covariant=True, bound="BaseType", default="BaseType")
+_Child: TypeAlias = tuple[Literal["child"], DTypeT_co]
+_Size: TypeAlias = tuple[Literal["size"], int]
+_ID_co = TypeVar("_ID_co", bound=str, default=str, covariant=True)
+
+
+class BaseType(Protocol[_ID_co]):
+    def __eq__(self, other: object) -> bool: ...
+    def __hash__(self) -> int: ...
+    @overload
+    def __init__(self, type_str: str, connection: DuckDBPyConnection) -> None: ...
+    @overload
+    def __init__(self, obj: object) -> None: ...
+    @property
+    def id(self) -> _ID_co: ...
+
+
+class _ParentType(BaseType[_ID_co], Protocol[_ID_co, T_co]):
+    @property
+    def children(self) -> T_co: ...
+
+
+class ArrayType(
+    _ParentType[Literal["array"], tuple[_Child[DTypeT_co], _Size]], Protocol[DTypeT_co]
+): ...
+
+
+class EnumType(
+    _ParentType[Literal["enum"], tuple[tuple[Literal["values"], list[str]]]], Protocol
+): ...
+
+
+class ListType(
+    _ParentType[Literal["list"], tuple[_Child[DTypeT_co]]], Protocol[DTypeT_co]
+):
+    @property
+    def child(self) -> DTypeT_co: ...
+
+
+class StructType(
+    _ParentType[Literal["struct"], Sequence[tuple[str, BaseType]]], Protocol
+):
+    def __getattr__(self, name: str) -> BaseType: ...
+    def __getitem__(self, name: str) -> BaseType: ...
+
+
+class DecimalType(
+    _ParentType[
+        Literal["decimal"],
+        tuple[tuple[Literal["precision"], int], tuple[Literal["scale"], int]],
+    ],
+    Protocol,
+): ...
+
+
+def has_children(
+    dtype: BaseType | _ParentType[_ID_co, T_co],
+) -> TypeIs[_ParentType[_ID_co, T_co]]:
+    """Using `_hasattr_static` returns `True` on any `DuckDBPyType.
+
+    The only way to be sure is forcing an exception.
+    """
+    import duckdb
+
+    try:
+        return hasattr(dtype, "children")
+    except duckdb.InvalidInputException:
+        return False
+
+
+def is_dtype_array(obj: BaseType) -> TypeIs[ArrayType]:
+    return obj.id == "array"
+
+
+def is_dtype_struct(obj: BaseType) -> TypeIs[StructType]:
+    return obj.id == "struct"
+
+
+def is_dtype_list(obj: BaseType) -> TypeIs[ListType]:
+    return obj.id == "list"
+
+
+def is_dtype_enum(obj: BaseType) -> TypeIs[EnumType]:
+    return obj.id == "enum"
+
+
+def is_dtype_timestamp_with_time_zone(
+    obj: BaseType,
+) -> TypeIs[BaseType[Literal["timestamp with time zone"]]]:
+    return obj.id == "timestamp with time zone"
+
+
+def is_dtype_decimal(obj: BaseType) -> TypeIs[DecimalType]:
+    return obj.id == "decimal"

narwhals/_duckdb/utils.py

@@ -1,31 +1,37 @@
 from __future__ import annotations
 
 from functools import lru_cache
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import duckdb
 from duckdb import Expression
 
-try:
-    import duckdb.sqltypes as duckdb_dtypes
-except ModuleNotFoundError:
-    # DuckDB pre 1.3
-    import duckdb.typing as duckdb_dtypes
-
-from narwhals._utils import Version, isinstance_or_issubclass, zip_strict
-from narwhals.exceptions import ColumnNotFoundError
+import narwhals._duckdb.typing as nw_dd_t
+from narwhals._utils import Implementation, Version, isinstance_or_issubclass, zip_strict
+from narwhals.exceptions import ColumnNotFoundError, UnsupportedDTypeError
 
 if TYPE_CHECKING:
     from collections.abc import Mapping, Sequence
 
     from duckdb import DuckDBPyRelation
+    from duckdb.sqltypes import DuckDBPyType
+    from typing_extensions import TypeAlias
 
     from narwhals._compliant.typing import CompliantLazyFrameAny
     from narwhals._duckdb.dataframe import DuckDBLazyFrame
     from narwhals._duckdb.expr import DuckDBExpr
     from narwhals.dtypes import DType
-    from narwhals.typing import IntoDType, TimeUnit
+    from narwhals.typing import IntoDType, NonNestedLiteral, TimeUnit
+
+Incomplete: TypeAlias = Any
 
+BACKEND_VERSION = Implementation.DUCKDB._backend_version()
+"""Static backend version for `duckdb`."""
+
+if TYPE_CHECKING or BACKEND_VERSION >= (1, 4):
+    from duckdb import sqltypes as duckdb_dtypes
+else:  # pragma: no cover
+    from duckdb import typing as duckdb_dtypes
 
 UNITS_DICT = {
     "y": "year",
@@ -45,8 +51,13 @@
 col = duckdb.ColumnExpression
 """Alias for `duckdb.ColumnExpression`."""
 
-lit = duckdb.ConstantExpression
-"""Alias for `duckdb.ConstantExpression`."""
+
+# TODO @dangotbanned: Raise an issue upstream on `Expression | str` too narrow
+def lit(value: Expression | NonNestedLiteral | Sequence[Any]) -> Expression:
+    """Alias for `duckdb.ConstantExpression`."""
+    lit_: Incomplete = duckdb.ConstantExpression
+    return lit_(value)
+
 
 when = duckdb.CaseExpression
 """Alias for `duckdb.CaseExpression`."""
@@ -55,6 +66,8 @@
 """Alias for `duckdb.FunctionExpression`."""
 
 
+# TODO @dangotbanned: Investigate `lhs: Expression | str | tuple[str]`
+# Seems incorrect
 def lambda_expr(
     params: str | Expression | tuple[Expression, ...], expr: Expression, /
 ) -> Expression:
@@ -68,7 +81,8 @@ def lambda_expr(
         msg = f"DuckDB>=1.2.0 is required for this operation. Found: DuckDB {duckdb.__version__}"
         raise NotImplementedError(msg) from exc
     args = (params,) if isinstance(params, Expression) else params
-    return LambdaExpression(args, expr)
+    lambda_expr_: Incomplete = LambdaExpression
+    return lambda_expr_(args, expr)
 
 
 def concat_str(*exprs: Expression, separator: str = "") -> Expression:
@@ -135,20 +149,27 @@ def time_zone(self) -> str:
 
 
 def native_to_narwhals_dtype(
-    duckdb_dtype: duckdb_dtypes.DuckDBPyType,
+    duckdb_dtype: nw_dd_t.BaseType, version: Version, deferred_time_zone: DeferredTimeZone
+) -> DType:
+    if nw_dd_t.has_children(duckdb_dtype) and not nw_dd_t.is_dtype_decimal(duckdb_dtype):
+        return _nested_native_to_narwhals_dtype(duckdb_dtype, version, deferred_time_zone)
+    if nw_dd_t.is_dtype_timestamp_with_time_zone(duckdb_dtype):
+        return version.dtypes.Datetime(time_zone=deferred_time_zone.time_zone)
+    return _non_nested_native_to_narwhals_dtype(duckdb_dtype.id, version)
+
+
+def _nested_native_to_narwhals_dtype(
+    duckdb_dtype: nw_dd_t._ParentType,
     version: Version,
     deferred_time_zone: DeferredTimeZone,
 ) -> DType:
-    duckdb_dtype_id = duckdb_dtype.id
     dtypes = version.dtypes
 
-    # Handle nested data types first
-    if duckdb_dtype_id == "list":
+    if nw_dd_t.is_dtype_list(duckdb_dtype):
         return dtypes.List(
             native_to_narwhals_dtype(duckdb_dtype.child, version, deferred_time_zone)
         )
-
-    if duckdb_dtype_id == "struct":
+    if nw_dd_t.is_dtype_struct(duckdb_dtype):
         children = duckdb_dtype.children
         return dtypes.Struct(
             [
@@ -159,36 +180,31 @@ def native_to_narwhals_dtype(
                 for child in children
             ]
         )
-
-    if duckdb_dtype_id == "array":
+    if nw_dd_t.is_dtype_array(duckdb_dtype):
         child, size = duckdb_dtype.children
         shape: list[int] = [size[1]]
 
-        while child[1].id == "array":
+        while nw_dd_t.is_dtype_array(child[1]):
             child, size = child[1].children
             shape.insert(0, size[1])
 
         inner = native_to_narwhals_dtype(child[1], version, deferred_time_zone)
         return dtypes.Array(inner=inner, shape=tuple(shape))
-
-    if duckdb_dtype_id == "enum":
+    if nw_dd_t.is_dtype_enum(duckdb_dtype):
         if version is Version.V1:
             return dtypes.Enum()  # type: ignore[call-arg]
         categories = duckdb_dtype.children[0][1]
         return dtypes.Enum(categories=categories)
-
-    if duckdb_dtype_id == "timestamp with time zone":
-        return dtypes.Datetime(time_zone=deferred_time_zone.time_zone)
-
-    return _non_nested_native_to_narwhals_dtype(duckdb_dtype_id, version)
+    # `MAP`, `UNION`
+    raise UnsupportedDTypeError(duckdb_dtype)
 
 
 def fetch_rel_time_zone(rel: duckdb.DuckDBPyRelation) -> str:
     result = rel.query(
         "duckdb_settings()", "select value from duckdb_settings() where name = 'TimeZone'"
     ).fetchone()
     assert result is not None  # noqa: S101
-    return result[0]  # type: ignore[no-any-return]
+    return result[0]
 
 
 @lru_cache(maxsize=16)
@@ -222,7 +238,7 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
 
 
 dtypes = Version.MAIN.dtypes
-NW_TO_DUCKDB_DTYPES: Mapping[type[DType], duckdb_dtypes.DuckDBPyType] = {
+NW_TO_DUCKDB_DTYPES: Mapping[type[DType], DuckDBPyType] = {
     dtypes.Float64: duckdb_dtypes.DOUBLE,
     dtypes.Float32: duckdb_dtypes.FLOAT,
     dtypes.Binary: duckdb_dtypes.BLOB,
@@ -241,7 +257,7 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
     dtypes.UInt64: duckdb_dtypes.UBIGINT,
     dtypes.UInt128: duckdb_dtypes.UHUGEINT,
 }
-TIME_UNIT_TO_TIMESTAMP: Mapping[TimeUnit, duckdb_dtypes.DuckDBPyType] = {
+TIME_UNIT_TO_TIMESTAMP: Mapping[TimeUnit, DuckDBPyType] = {
     "s": duckdb_dtypes.TIMESTAMP_S,
     "ms": duckdb_dtypes.TIMESTAMP_MS,
     "us": duckdb_dtypes.TIMESTAMP,
@@ -252,7 +268,7 @@ def _non_nested_native_to_narwhals_dtype(duckdb_dtype_id: str, version: Version)
 
 def narwhals_to_native_dtype(  # noqa: PLR0912, C901
     dtype: IntoDType, version: Version, deferred_time_zone: DeferredTimeZone
-) -> duckdb_dtypes.DuckDBPyType:
+) -> DuckDBPyType:
     dtypes = version.dtypes
     base_type = dtype.base_type()
     if duckdb_type := NW_TO_DUCKDB_DTYPES.get(base_type):