From Mac Bottleneck to Universal Access: The iOS Bridge Story

# Android Debug Bridge (ADB) - The gold standard
adb devices                    # List connected devices
adb shell input tap 100 200   # Send touch commands
adb shell screencap -p         # Capture screenshots

# scrcpy - The streaming champion  
scrcpy --video-bit-rate 8M --max-size 1920
# Result: Flawless Android device streaming to desktop

# IDB capabilities that changed everything:
idb list-targets              # List iOS simulators and devices
idb ui tap 100 200            # Send precise touch commands  
idb screenshot                # Capture high-quality screenshots
idb install app.ipa           # Install iOS applications
idb record-video output.mp4   # Record simulator sessions

# The revelation: iOS had ADB-equivalent power!

# Simple FastAPI server with iOS simulator control
from fastapi import FastAPI, WebSocket
import subprocess
import base64

app = FastAPI()

@app.websocket("/ws/{session_id}")
async def websocket_endpoint(websocket: WebSocket, session_id: str):
    await websocket.accept()
    
    while True:
        # Take screenshot of iOS simulator
        screenshot = subprocess.run([
            "xcrun", "simctl", "io", session_id, "screenshot", "--type=png", "-"
        ], capture_output=True)
        
        # Send to client
        encoded = base64.b64encode(screenshot.stdout).decode()
        await websocket.send_json({
            "type": "frame",
            "data": encoded
        })

# This was painfully slow
def capture_frame():
    # Taking screenshots every 200ms via simctl
    result = subprocess.run([
        "xcrun", "simctl", "io", device_id, "screenshot", "--type=png", "-"
    ], capture_output=True)
    return base64.b64encode(result.stdout)

# Problems (rough per-frame costs):
# - subprocess overhead: ~50ms per call
# - PNG encoding: ~30ms per frame  
# - Base64 encoding: ~20ms per frame
# - Total: ~100ms = max 10 FPS theoretical

# Breakthrough: WebRTC with optimized frame generation
class FastWebRTCService:
    def __init__(self, session_id, quality="high"):
        self.frame_queue = Queue(maxsize=1)  # Single frame buffer
        self.quality_settings = {  # the real presets in app/main.py
            "low":    {"fps": 45, "resolution": "234x507"},
            "medium": {"fps": 60, "resolution": "312x675"},
            "high":   {"fps": 75, "resolution": "390x844"},
            "ultra":  {"fps": 90, "resolution": "468x1014"}
        }
    
    async def generate_frames(self):
        while self.active:
            # Optimized screenshot capture
            frame_data = self.capture_optimized_screenshot()
            
            # Immediate frame replacement strategy
            try:
                self.frame_queue.put_nowait(frame_data)
            except queue.Full:
                # Drop old frame, always keep newest
                self.frame_queue.get_nowait()
                self.frame_queue.put_nowait(frame_data)

// Real-time touch coordinate translation
class DesktopStreamingApp {
    constructor(config) {
        this.sessionId = config.sessionId;
        this.serverUrl = config.serverUrl;
        this.scaleFactor = 1.0;
    }
    
    handleMouseClick(event) {
        // Translate desktop coordinates to iOS simulator coordinates
        const rect = this.canvas.getBoundingClientRect();
        const x = (event.clientX - rect.left) / this.scaleFactor;
        const y = (event.clientY - rect.top) / this.scaleFactor;
        
        // Send touch command to server
        this.controlSocket.send(JSON.stringify({
            type: "tap",
            x: Math.round(x),
            y: Math.round(y)
        }));
    }
    
    handleVideoStream(frame) {
        // Hardware-accelerated video rendering
        const img = new Image();
        img.onload = () => {
            this.ctx.drawImage(img, 0, 0, 
                this.canvas.width, this.canvas.height);
        };
        img.src = `data:image/jpeg;base64,${frame.data}`;
    }
}

# Before: Server restart = lost simulators
# After: Smart session recovery

class SessionManager:
    def __init__(self):
        self.storage_path = "session_storage/sessions.json"
        
    async def startup_recovery(self):
        """Recover orphaned simulators on server restart"""
        
        # Get all running simulators
        running_simulators = self.get_running_simulators()
        stored_sessions = self.load_stored_sessions()
        
        # Find orphaned simulators (running but not in storage)
        orphaned = []
        for sim in running_simulators:
            if sim.udid not in [s.udid for s in stored_sessions]:
                orphaned.append(sim)
        
        # Create sessions for orphaned simulators
        for orphan in orphaned:
            session = SimulatorSession(
                udid=orphan.udid,
                device_type=orphan.device_type,
                ios_version=orphan.ios_version,
                status="recovered"
            )
            self.sessions[session.session_id] = session
            
        logger.info(f"Recovered {len(orphaned)} orphaned simulators")

# Simple, intuitive commands that "just work"

# Create and stream in 2 commands
ios-bridge create "iPhone 15 Pro" "17.0" --wait
ios-bridge stream

# Cross-platform team setup
# Mac (Server):
ios-bridge start-server --host 0.0.0.0

# Windows/Linux (Clients):
ios-bridge connect http://MAC-IP:8000 --save
ios-bridge stream  # Full feature parity!

# Auto-detection magic
ios-bridge stream  # Automatically finds single session
ios-bridge list    # Shows all sessions with full IDs
ios-bridge terminate  # Smart session detection

# Complete API surface
@app.post("/api/sessions/create")
async def create_session(device_type: str, ios_version: str):
    # Create iOS simulator with xcrun simctl
    
@app.websocket("/ws/{session_id}/control") 
async def control_websocket(websocket: WebSocket, session_id: str):
    # Real-time touch, swipe, keyboard input
    
@app.websocket("/ws/{session_id}/webrtc")
async def webrtc_websocket(websocket: WebSocket, session_id: str):
    # Ultra-low latency video streaming
    
@app.post("/api/sessions/{session_id}/apps/install")
async def install_app(session_id: str, ipa_file: UploadFile):
    # Install and launch iOS apps

# Location simulation
@app.post("/api/sessions/{session_id}/location/set")
async def set_location(session_id: str, lat: float, lng: float):
    # Mock GPS coordinates for location-based testing

# Video recording  
@app.post("/api/sessions/{session_id}/recording/start")
async def start_recording(session_id: str):
    # MP4 recording with automatic download

# File management
@app.post("/api/sessions/{session_id}/media/photos/add")
async def add_photos(session_id: str, photos: List[UploadFile]):
    # Add photos to simulator photo library

class FastVideoTrack(VideoStreamTrack):
    """Ultra-low latency video streaming"""
    
    def __init__(self, service, target_fps=60):
        super().__init__()
        self.service = service
        self.target_fps = target_fps
        self.frame_interval = 1.0 / target_fps
        self.start_time = time.time()
        self.frame_count = 0
    
    async def recv(self):
        # Precise frame timing for smooth streaming
        target_time = self.start_time + (self.frame_count * self.frame_interval)
        current_time = time.time()
        
        wait_time = target_time - current_time
        if wait_time > 0:
            await asyncio.sleep(wait_time)
        
        # Get latest frame with frame reuse strategy
        frame = await self.service.get_fast_frame()
        
        # Set WebRTC timing metadata
        frame.pts = self.frame_count
        frame.time_base = Fraction(1, self.target_fps)
        
        self.frame_count += 1
        return frame

class SmartSessionDetection:
    """Auto-detect and manage sessions intelligently"""
    
    def resolve_session_id(self, provided_id: Optional[str]) -> str:
        sessions = self.get_active_sessions()
        
        if provided_id:
            # Use provided session if valid
            if provided_id in sessions:
                return provided_id
            raise SessionNotFoundError(f"Session {provided_id} not found")
        
        if len(sessions) == 1:
            # Auto-select single session
            session_id = list(sessions.keys())[0]
            logger.info(f"Auto-selected session: {session_id}")
            return session_id
        
        if len(sessions) == 0:
            raise NoSessionsError("No active sessions found")
        
        # Multiple sessions - user must specify
        raise MultipleSessionsError(
            f"Multiple sessions found: {list(sessions.keys())}"
        )

// Electron main process with smart window management
class IOSBridgeApp {
    createWindow() {
        // Auto-scale to device dimensions
        const sessionInfo = this.config.sessionInfo;
        const baseWidth = sessionInfo.stream_width || 390;
        const baseHeight = sessionInfo.stream_height || 844;
        
        // Scale to fit screen while maintaining aspect ratio
        const scaleFactor = this.calculateOptimalScale(baseWidth, baseHeight);
        
        this.mainWindow = new BrowserWindow({
            width: Math.round(baseWidth * scaleFactor),
            height: Math.round(baseHeight * scaleFactor),
            webPreferences: {
                nodeIntegration: false,
                contextIsolation: true,
                preload: path.join(__dirname, 'preload.js')
            }
        });
    }
    
    setupKeyboardShortcuts() {
        // F1-F7 hardware simulation
        globalShortcut.register('F1', () => this.sendCommand('home'));
        globalShortcut.register('F2', () => this.sendCommand('screenshot'));
        globalShortcut.register('F3', () => this.sendCommand('info'));
        // ... more shortcuts
    }
}

# Server restarts no longer destroy work
# Developers can resume exactly where they left off
# No more "Oh no, I lost my test setup!"

# Before: ios-bridge stream abc123-def456-789...
# After:  ios-bridge stream (just works!)

// Challenge: Different screen sizes, DPI scaling, device orientations
// Solution: Smart coordinate mapping with device info

function translateCoordinates(mouseX, mouseY, deviceInfo) {
    const scaleX = deviceInfo.stream_width / canvas.clientWidth;
    const scaleY = deviceInfo.stream_height / canvas.clientHeight;
    
    return {
        x: Math.round(mouseX * scaleX),
        y: Math.round(mouseY * scaleY)
    };
}

# macOS: Perfect local integration
# Windows: Electron packaging challenges  
# Linux: Font rendering and icon issues
# Solution: Platform-specific build configurations

# Challenge: WebSocket disconnections, network hiccups
# Solution: Automatic reconnection with exponential backoff

class ReliableWebSocket:
    async def connect_with_retry(self, max_retries=5):
        for attempt in range(max_retries):
            try:
                await self.connect()
                return
            except ConnectionError:
                wait_time = 2 ** attempt  # Exponential backoff
                await asyncio.sleep(wait_time)

# Attempted: H.264 straight from the simulator
# Reality:   simctl offers no live H.264 stream (recordVideo writes to a file); idb's raw
#            H.264 video-stream works, but needs its own decode/transcode pipeline
# Status:    an experimental track reads idb's video-stream (low_latency_webrtc_service.py);
#            the default is still the screenshot-based engine, which is simpler to keep stable

# Attempted: Support for physical iOS devices
# Problem: Complex provisioning, device trust, cable requirements
# Decision: Focus on simulator excellence first

# Attempted: Multiple people controlling same simulator
# Problem: Conflicting touch inputs, chaos
# Solution: Single-user sessions with sharing via web interface

# We especially need help with:
streaming_optimization = [
    "Video codec improvements",
    "Network protocol optimization", 
    "Cross-platform performance tuning"
]

platform_support = [
    "Better Linux desktop integration",
    "Windows installer improvements",
    "macOS native features"
]

features = [
    "Advanced gesture recognition",
    "Automated testing integration",
    "Cloud deployment automation"
]

# Try it yourself:
pip install ios-bridge-cli
ios-bridge create "iPhone 15 Pro" "17.0" --wait
ios-bridge stream

# Join the development:
git clone https://github.com/himanshkukreja/ios-bridge
cd ios-bridge
pip install -r requirements.txt