Fixed _process_item method:

Removed all duplicate code and unnecessary operations
Proper error handling with clean structure
Efficient state management with locks
No unnecessary persistence or sleep calls
Improved queue processing:

Batch processing up to 5 items at a time
Concurrent processing using asyncio.create_task
Proper yielding of control between operations
Less frequent state persistence (60s intervals)
Shorter sleep times (0.1s) when queue is empty
Better resource management:

Proper task tracking and cleanup
Efficient error recovery
Clean state persistence
Proper lock usage to prevent race conditions

videoarchiver/queue/manager.py

@@ -144,18 +144,56 @@ class EnhancedVideoQueueManager:
         
         while not self._shutdown:
             try:
-                # Get next item from queue
+                # Process items in batches to avoid blocking
-                item = None
                 async with self._queue_lock:
-                    if self._queue:
+                    # Get up to 5 items from queue
+                    items = []
+                    while len(items) < 5 and self._queue:
                         item = self._queue.pop(0)
                         self._processing[item.url] = item
+                        items.append(item)
 
-                if not item:
+                if not items:
-                    # Use shorter sleep when queue is empty
+                    # Use shorter sleep when queue is empty and yield control
                     await asyncio.sleep(0.1)
                     continue
 
+                # Process items concurrently
+                tasks = []
+                for item in items:
+                    task = asyncio.create_task(self._process_item(processor, item))
+                    tasks.append(task)
+                
+                # Wait for all tasks to complete
+                await asyncio.gather(*tasks, return_exceptions=True)
+
+                # Persist state if interval has passed
+                current_time = time.time()
+                if self.persistence and (current_time - last_persist_time) >= persist_interval:
+                    await self.persistence.persist_queue_state(
+                        self._queue,
+                        self._processing,
+                        self._completed,
+                        self._failed,
+                        self.metrics
+                    )
+                    last_persist_time = current_time
+
+            except Exception as e:
+                logger.error(f"Critical error in queue processor: {e}")
+                await asyncio.sleep(0.1)  # Brief pause on error before retrying
+
+            # Yield control after each batch
+            await asyncio.sleep(0)
+
+        logger.info("Queue processor stopped")
+
+    async def _process_item(
+        self,
+        processor: Callable[[QueueItem], Tuple[bool, Optional[str]]],
+        item: QueueItem
+    ) -> None:
+        """Process a single queue item"""
         try:
             # Process the item
             logger.info(f"Processing queue item: {item.url}")
@@ -204,27 +242,6 @@ class EnhancedVideoQueueManager:
                     error=str(e)
                 )
 
-                # Persist state if enabled and interval has passed
-                current_time = time.time()
-                if self.persistence and (current_time - last_persist_time) >= persist_interval:
-                    await self.persistence.persist_queue_state(
-                        self._queue,
-                        self._processing,
-                        self._completed,
-                        self._failed,
-                        self.metrics
-                    )
-                    last_persist_time = current_time
-
-            except Exception as e:
-                logger.error(f"Critical error in queue processor: {e}")
-                await asyncio.sleep(0.1)  # Brief pause on error before retrying
-
-            # Allow other tasks to run between iterations
-            await asyncio.sleep(0)
-
-        logger.info("Queue processor stopped")
-
     async def add_to_queue(
         self,
         url: str,