Issue with resume training model

I was training my model from scratch using this code (set resolution 896 in both model and train config). ``` from rfdetr import RFDETRMedium from rfdetr.datasets.aug_config import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL model = RFDETRMedium(resolution=896) ROOT_PATH = '../../' model.train( dataset_dir=ROOT_PATH + 'data/high_res_ds_split', resolution=896, epochs=100, batch_size=4, grad_accum_steps=2, aug_config=AUG_AGGRESSIVE, progress_bar=True, output_dir='finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026', checkpoint_interval=1, save_dataset_grids=True, tensorboard=True ) ``` After model 44 my train was stopped (it was expected). Now I want to resume my training from latest checkpoint (checkpoint_44.ckpt). But when i am trying to do this i encounter few errors: 1) If I use this code: ``` from rfdetr import RFDETRMedium from rfdetr.datasets.aug_config import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL model = RFDETRMedium(resolution=896) ROOT_PATH = '../../' model.train( dataset_dir=ROOT_PATH + 'data/high_res_ds_split', resolution=896, epochs=100, batch_size=4, grad_accum_steps=2, aug_config=AUG_AGGRESSIVE, progress_bar=True, output_dir='finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026', checkpoint_interval=1, save_dataset_grids=True, tensorboard=True, resume="finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/checkpoint_44.ckpt", devices="auto", ) ``` I am getting this type of output: ``` [2026-04-25 14:02:00] [INFO] rf-detr - File rf-detr-medium.pth already exists with correct MD5 hash. [2026-04-25 14:02:00] [WARNING] rf-detr - Using a different number of positional encodings than DINOv2, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:02:00] [WARNING] rf-detr - Using patch size 16 instead of 14, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:02:02] [INFO] rf-detr - File rf-detr-medium.pth already exists with correct MD5 hash. --------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) Cell In[4], line 6 3 from rfdetr import RFDETRMedium 4 from rfdetr.datasets.aug_config import AUG_CONSERVATIVE, AUG_AGGRESSIVE, AUG_AERIAL, AUG_INDUSTRIAL ----> 6 model = RFDETRMedium(resolution=896) 8 ROOT_PATH = '../../' 10 model.train( 11 dataset_dir=ROOT_PATH + 'data/high_res_ds_split', 12 resolution=896, (...) 23 devices="auto", 24 ) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/rfdetr/detr.py:382, in RFDETR.__init__(self, **kwargs) 380 self.model_config = self.get_model_config(**kwargs) 381 self.maybe_download_pretrain_weights() --> 382 self.model = self.get_model(self.model_config) 383 self.callbacks = defaultdict(list) 385 self.model.inference_model = None File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/rfdetr/detr.py:1027, in RFDETR.get_model(self, config) 1016 def get_model(self, config: ModelConfig) -> "ModelContext": 1017 """Retrieve a model context from the provided architecture configuration. 1018 1019 Args: (...) 1025 1026 """ -> 1027 return _build_model_context(config) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/rfdetr/detr.py:236, in _build_model_context(model_config) 234 class_names: List[str] = [] 235 if args.pretrain_weights is not None: --> 236 class_names = _load_pretrain_weights_into(nn_model, args) 238 if args.backbone_lora: 239 _apply_lora_to(nn_model) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/rfdetr/detr.py:169, in _load_pretrain_weights_into(nn_model, args) 166 if any(name.endswith(x) for x in query_param_names): 167 checkpoint["model"][name] = checkpoint["model"][name][:num_desired_queries] --> 169 nn_model.load_state_dict(checkpoint["model"], strict=False) 171 # Only reinitialize back to configured size when intentionally reducing a 172 # larger pretrain checkpoint to fewer task-specific classes. 173 if checkpoint_num_classes < configured_num_classes_plus_bg and user_overrode_default_num_classes: File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/torch/nn/modules/module.py:2581, in Module.load_state_dict(self, state_dict, strict, assign) 2573 error_msgs.insert( 2574 0, 2575 "Missing key(s) in state_dict: {}. ".format( 2576 ", ".join(f'"{k}"' for k in missing_keys) 2577 ), 2578 ) 2580 if len(error_msgs) > 0: -> 2581 raise RuntimeError( 2582 "Error(s) in loading state_dict for {}:\n\t{}".format( 2583 self.__class__.__name__, "\n\t".join(error_msgs) 2584 ) 2585 ) 2586 return _IncompatibleKeys(missing_keys, unexpected_keys) RuntimeError: Error(s) in loading state_dict for LWDETR: size mismatch for backbone.0.encoder.encoder.embeddings.position_embeddings: copying a param with shape torch.Size([1, 1297, 384]) from checkpoint, the shape in current model is torch.Size([1, 3137, 384]). ``` 2) If i will specify `pretrain_weights=None`, then i get this output: ``` [2026-04-25 14:04:54] [WARNING] rf-detr - Using a different number of positional encodings than DINOv2, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:04:54] [WARNING] rf-detr - Using patch size 16 instead of 14, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:05:01] [WARNING] rf-detr - Using a different number of positional encodings than DINOv2, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:05:01] [WARNING] rf-detr - Using patch size 16 instead of 14, which means we're not loading DINOv2 backbone weights. This is not a problem if finetuning a pretrained RF-DETR model. [2026-04-25 14:05:01] [INFO] rf-detr - Building Roboflow train dataset with square resize at resolution 896 [2026-04-25 14:05:01] [INFO] rf-detr - Using multi-scale training with square resize and scales: [1056] [2026-04-25 14:05:01] [INFO] rf-detr - Built 1 Albumentations transforms from config [2026-04-25 14:05:01] [INFO] rf-detr - Built 5 Albumentations transforms from config loading annotations into memory... Done (t=5.24s) creating index... index created! [2026-04-25 14:05:07] [INFO] rf-detr - Building Roboflow val dataset with square resize at resolution 896 [2026-04-25 14:05:07] [INFO] rf-detr - Using multi-scale training with square resize and scales: [1056] [2026-04-25 14:05:07] [INFO] rf-detr - Built 1 Albumentations transforms from config loading annotations into memory... Done (t=1.15s) creating index... index created! UserWarning: This DataLoader will create 2 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. [2026-04-25 14:05:13] [INFO] rf-detr - Saved train grids with augmented images to: /home/alievrr/CAR_DETECTION/notebooks/train/finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/dataset_grids UserWarning: This DataLoader will create 2 worker processes in total. Our suggested max number of worker in current system is 1, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary. [2026-04-25 14:05:17] [INFO] rf-detr - Saved val grids with augmented images to: /home/alievrr/CAR_DETECTION/notebooks/train/finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/dataset_grids Trainer will use only 1 of 2 GPUs because it is running inside an interactive / notebook environment. You may try to set `Trainer(devices=2)` but please note that multi-GPU inside interactive / notebook environments is considered experimental and unstable. Your mileage may vary. PossibleUserWarning: The `srun` command is available on your system but is not used. HINT: If your intention is to run Lightning on SLURM, prepend your python command with `srun` like so: srun python /home/alievrr/.local/share/mamba/envs/rfdetr_311/lib ... Using bfloat16 Automatic Mixed Precision (AMP) GPU available: True (cuda), used: True TPU available: False, using: 0 TPU cores 💡 Tip: For seamless cloud logging and experiment tracking, try installing [litlogger](https://pypi.org/project/litlogger/) to enable LitLogger, which logs metrics and artifacts automatically to the Lightning Experiments platform. UserWarning: Experiment logs directory /home/alievrr/CAR_DETECTION/notebooks/train/finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/ exists and is not empty. Previous log files in this directory will be deleted when the new ones are saved! UserWarning: Checkpoint directory /home/alievrr/CAR_DETECTION/notebooks/train/finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026 exists and is not empty. Restoring states from the checkpoint path at finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/checkpoint_44.ckpt --------------------------------------------------------------------------- RuntimeError Traceback (most recent call last) Cell In[6], line 8 4 model = RFDETRMedium(resolution=896, pretrain_weights=None) 6 ROOT_PATH = '../../' ----> 8 model.train( 9 dataset_dir=ROOT_PATH + 'data/high_res_ds_split', 10 resolution=896, 11 epochs=100, 12 batch_size=4, 13 grad_accum_steps=2, 14 aug_config=AUG_AGGRESSIVE, 15 progress_bar=True, 16 output_dir='finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026', 17 checkpoint_interval=1, 18 save_dataset_grids=True, 19 tensorboard=True, 20 resume="finetuning_artifacts_rfdetr/res=896_batch=8_grad=2_augs=aggressive_20.04.2026/checkpoint_44.ckpt", 21 devices="auto", 22 ) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/rfdetr/detr.py:635, in RFDETR.train(self, **kwargs) 633 trainer_kwargs["devices"] = _devices 634 trainer = build_trainer(config, self.model_config, **trainer_kwargs) --> 635 trainer.fit(module, datamodule, ckpt_path=config.resume or None) 637 # Sync the trained weights back so predict() / export() see the updated model. 638 self.model.model = module.model File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py:584, in Trainer.fit(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path, weights_only) 582 self.training = True 583 self.should_stop = False --> 584 call._call_and_handle_interrupt( 585 self, 586 self._fit_impl, 587 model, 588 train_dataloaders, 589 val_dataloaders, 590 datamodule, 591 ckpt_path, 592 weights_only, 593 ) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/call.py:49, in _call_and_handle_interrupt(trainer, trainer_fn, *args, **kwargs) 47 if trainer.strategy.launcher is not None: 48 return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs) ---> 49 return trainer_fn(*args, **kwargs) 51 except _TunerExitException: 52 _call_teardown_hook(trainer) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py:630, in Trainer._fit_impl(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path, weights_only) 623 download_model_from_registry(ckpt_path, self) 624 ckpt_path = self._checkpoint_connector._select_ckpt_path( 625 self.state.fn, 626 ckpt_path, 627 model_provided=True, 628 model_connected=self.lightning_module is not None, 629 ) --> 630 self._run(model, ckpt_path=ckpt_path, weights_only=weights_only) 632 assert self.state.stopped 633 self.training = False File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/trainer.py:1046, in Trainer._run(self, model, ckpt_path, weights_only) 1044 if not self.strategy.restore_checkpoint_after_setup: 1045 log.debug(f"{self.__class__.__name__}: restoring module and callbacks from checkpoint path: {ckpt_path}") -> 1046 self._checkpoint_connector._restore_modules_and_callbacks(ckpt_path, weights_only) 1048 # reset logger connector 1049 self._logger_connector.reset_results() File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py:411, in _CheckpointConnector._restore_modules_and_callbacks(self, checkpoint_path, weights_only) 406 def _restore_modules_and_callbacks( 407 self, checkpoint_path: Optional[_PATH] = None, weights_only: Optional[bool] = None 408 ) -> None: 409 # restore modules after setup 410 self.resume_start(checkpoint_path, weights_only=weights_only) --> 411 self.restore_model() 412 self.restore_datamodule() 413 self.restore_callbacks() File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/trainer/connectors/checkpoint_connector.py:286, in _CheckpointConnector.restore_model(self) 283 call._call_lightning_module_hook(self.trainer, "on_load_checkpoint", self._loaded_checkpoint) 285 # restore model state_dict --> 286 self.trainer.strategy.load_model_state_dict( 287 self._loaded_checkpoint, 288 strict=self.trainer.lightning_module.strict_loading, 289 ) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/pytorch_lightning/strategies/strategy.py:372, in Strategy.load_model_state_dict(self, checkpoint, strict) 370 def load_model_state_dict(self, checkpoint: Mapping[str, Any], strict: bool = True) -> None: 371 assert self.lightning_module is not None --> 372 self.lightning_module.load_state_dict(checkpoint["state_dict"], strict=strict) File ~/.local/share/mamba/envs/rfdetr_311/lib/python3.11/site-packages/torch/nn/modules/module.py:2581, in Module.load_state_dict(self, state_dict, strict, assign) 2573 error_msgs.insert( 2574 0, 2575 "Missing key(s) in state_dict: {}. ".format( 2576 ", ".join(f'"{k}"' for k in missing_keys) 2577 ), 2578 ) 2580 if len(error_msgs) > 0: -> 2581 raise RuntimeError( 2582 "Error(s) in loading state_dict for {}:\n\t{}".format( 2583 self.__class__.__name__, "\n\t".join(error_msgs) 2584 ) 2585 ) 2586 return _IncompatibleKeys(missing_keys, unexpected_keys) RuntimeError: Error(s) in loading state_dict for RFDETRModelModule: size mismatch for model.backbone.0.encoder.encoder.embeddings.position_embeddings: copying a param with shape torch.Size([1, 1297, 384]) from checkpoint, the shape in current model is torch.Size([1, 3137, 384]). ``` So, how am i supposed to resume training from my latest checkpoint? **Versions** OS: CentOS Linux release 7.9.2009 (Core) Python: 3.11.14 rfdetr: 1.6.5