How to Schedule Recurring Google Maps Scrapes with Webhooks

import os
import requests

# Bolt Scraper API Endpoint & Credentials
API_URL = "https://boltscraper.com/api/v1/maps/jobs"
API_KEY = os.getenv("BOLT_API_KEY", "bs_YOUR_API_KEY")

# Webhook receiver that will handle the completed job notification
WEBHOOK_CALLBACK = "https://api.yourdomain.com/webhooks/bolt-scraper"

def trigger_weekly_scrape():
    headers = {
        "Authorization": f"Bearer {API_KEY}",
        "Content-Type": "application/json"
    }
    
    payload = {
        "keywords": [
            "HVAC contractors in Charlotte, NC",
            "HVAC contractors in Raleigh, NC",
            "HVAC contractors in Greensboro, NC"
        ],
        "scrape_websites": True,      # Crawls websites for contact emails and social media
        "ultra_mode": True,            # Optional: up to 3x more leads (Standard mode already bypasses the 120-lead cap)
        "webhook_url": WEBHOOK_CALLBACK
    }
    
    response = requests.post(API_URL, headers=headers, json=payload)
    data = response.json()
    
    if response.status_code == 200 and data.get("success"):
        print(f"Scrape initiated! Job ID: {data['job_id']}")
        print(f"Keywords queued: {data['keywords_count']}")
    else:
        print(f"Failed to start job: {data}")

if __name__ == "__main__":
    trigger_weekly_scrape()

# Run weekly Google Maps discovery every Monday at 07:00 UTC
0 7 * * 1 /usr/bin/python3 /opt/leads/schedule_gmap_scrape.py >> /var/log/lead_scrape.log 2>&1

const express = require('express');
const crypto = require('crypto');
const axios = require('axios');

const app = express();
const WEBHOOK_SECRET = process.env.BOLT_WEBHOOK_SECRET || 'your_whsec_secret';

// Preserve raw body buffer for HMAC signature verification
app.use(express.json({
    verify: (req, res, buf) => { req.rawBody = buf; }
}));

app.post('/webhooks/bolt-scraper', async (req, res) => {
    const signature = req.headers['x-bolt-signature'];
    const eventType = req.headers['x-bolt-event'];

    // 1. Verify HMAC SHA-256 Signature (Optional but recommended)
    if (signature && WEBHOOK_SECRET) {
        const hmac = crypto.createHmac('sha256', WEBHOOK_SECRET);
        const digest = 'sha256=' + hmac.update(req.rawBody).digest('hex');
        if (signature !== digest) {
            return res.status(401).send('Invalid webhook signature');
        }
    }

    // 2. Handle gmap_job.completed event
    if (eventType === 'gmap_job.completed') {
        const { job_id, total_leads, download_csv_url, leads } = req.body;
        
        console.log(`Job ${job_id} finished with ${total_leads} unique business leads.`);
        
        // Process inline leads or download the complete CSV export
        if (download_csv_url) {
            console.log(`Downloading full CSV from: ${download_csv_url}`);
            // Ingest CSV directly into your master database or CRM
            await syncLeadsToDatabase(download_csv_url);
        }
    }

    // Acknowledge receipt promptly with HTTP 200
    res.status(200).json({ received: true });
});

async function syncLeadsToDatabase(csvUrl) {
    // Fetch CSV stream and upsert records into Postgres / Airtable
    const response = await axios.get(csvUrl, { responseType: 'stream' });
    // Records are already deduplicated in cloud by Bolt Scraper!
}

app.listen(3000, () => console.log('Webhook receiver listening on port 3000'));

-- Create leads table with unique place_id constraint
CREATE TABLE IF NOT EXISTS gmap_leads (
    place_id VARCHAR(255) PRIMARY KEY,
    business_name VARCHAR(255) NOT NULL,
    phone VARCHAR(50),
    website TEXT,
    email VARCHAR(255),
    address TEXT,
    rating NUMERIC(2, 1),
    reviews_count INTEGER,
    last_scraped_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);

-- Safe upsert: Insert fresh leads or update review ratings on existing leads
INSERT INTO gmap_leads (place_id, business_name, phone, website, email, rating, reviews_count)
VALUES ('ChIJN1t_tDeuEmsRUsoyG83frY4', 'Alpha HVAC Services', '+1-704-555-0199', 'https://alphahvac.com', '[email protected]', 4.8, 142)
ON CONFLICT (place_id) 
DO UPDATE SET 
    rating = EXCLUDED.rating,
    reviews_count = EXCLUDED.reviews_count,
    last_scraped_at = CURRENT_TIMESTAMP;