Use deformable_detr kernel from the Hub

Remove the `deformable_detr` kernel from `kernels/` and use the
pre-built kernel from the Hub instead.

Author: danieldk

setup.py

@@ -129,6 +129,7 @@
     # Keras pin - this is to make sure Keras 3 doesn't destroy us. Remove or change when we have proper support.
     "keras>2.9,<2.16",
     "keras-nlp>=0.3.1,<0.14.0",  # keras-nlp 0.14 doesn't support keras 2, see pin on keras.
+    "kernels>=0.3.2,<0.4",
     "librosa",
     "natten>=0.14.6,<0.15.0",
     "nltk<=3.8.1",
@@ -411,6 +412,7 @@ def run(self):
     "filelock",
     "huggingface-hub",
     "importlib_metadata",
+    "kernels",
     "numpy",
     "packaging",
     "protobuf",
@@ -432,6 +434,7 @@ def run(self):
 install_requires = [
     deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
     deps["huggingface-hub"],
+    deps["kernels"],  # download kernels from the Hub
     deps["numpy"],
     deps["packaging"],  # utilities from PyPA to e.g., compare versions
     deps["pyyaml"],  # used for the model cards metadata

src/transformers/dependency_versions_check.py

@@ -29,6 +29,7 @@
     "requests",
     "packaging",
     "filelock",
+    "kernels",
     "numpy",
     "tokenizers",
     "huggingface-hub",

src/transformers/dependency_versions_table.py

@@ -35,6 +35,7 @@
     "kenlm": "kenlm",
     "keras": "keras>2.9,<2.16",
     "keras-nlp": "keras-nlp>=0.3.1,<0.14.0",
+    "kernels": "kernels>=0.3.2,<0.4",
     "librosa": "librosa",
     "natten": "natten>=0.14.6,<0.15.0",
     "nltk": "nltk<=3.8.1",

src/transformers/integrations/__init__.py

@@ -70,6 +70,12 @@
         "replace_with_higgs_linear",
     ],
     "hqq": ["prepare_for_hqq_linear"],
+    "hub_kernels": [
+        "LayerRepository",
+        "register_kernel_mapping",
+        "replace_kernel_forward_from_hub",
+        "use_kernel_forward_from_hub",
+    ],
     "integration_utils": [
         "INTEGRATION_TO_CALLBACK",
         "AzureMLCallback",
@@ -198,6 +204,12 @@
     )
     from .higgs import HiggsLinear, dequantize_higgs, quantize_with_higgs, replace_with_higgs_linear
     from .hqq import prepare_for_hqq_linear
+    from .hub_kernels import (
+        LayerRepository,
+        register_kernel_mapping,
+        replace_kernel_forward_from_hub,
+        use_kernel_forward_from_hub,
+    )
     from .integration_utils import (
         INTEGRATION_TO_CALLBACK,
         AzureMLCallback,

src/transformers/integrations/hub_kernels.py

@@ -0,0 +1,28 @@
+from typing import Dict, Union
+
+from kernels import (
+    Device,
+    LayerRepository,
+    register_kernel_mapping,
+    replace_kernel_forward_from_hub,
+    use_kernel_forward_from_hub,
+)
+
+
+_KERNEL_MAPPING: Dict[str, Dict[Union[Device, str], LayerRepository]] = {
+    "MultiScaleDeformableAttention": {
+        "cuda": LayerRepository(
+            repo_id="kernels-community/deformable-detr",
+            layer_name="MultiScaleDeformableAttention",
+        )
+    }
+}
+
+register_kernel_mapping(_KERNEL_MAPPING)
+
+__all__ = [
+    "LayerRepository",
+    "use_kernel_forward_from_hub",
+    "register_kernel_mapping",
+    "replace_kernel_forward_from_hub",
+]