crawlee 1.1.1b8__tar.gz → 1.1.1b9__tar.gz

{crawlee-1.1.1b8 → crawlee-1.1.1b9}/.github/workflows/build_and_deploy_docs.yaml

@@ -67,6 +67,10 @@ jobs:
         uses: actions/deploy-pages@v4
 
       - name: Invalidate CloudFront cache
-        run: gh workflow run invalidate.yaml --repo apify/apify-docs-private
+        run: |
+          gh workflow run invalidate-cloudfront.yml \
+            --repo apify/apify-docs-private \
+            --field deployment=crawlee-web
+          echo "✅ CloudFront cache invalidation workflow triggered successfully"
         env:
           GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

{crawlee-1.1.1b8 → crawlee-1.1.1b9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.1.1b8
+Version: 1.1.1b9
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog

{crawlee-1.1.1b8 → crawlee-1.1.1b9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.1.1b8"
+version = "1.1.1b9"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

{crawlee-1.1.1b8 → crawlee-1.1.1b9}/uv.lock

@@ -691,7 +691,7 @@ toml = [
 
 [[package]]
 name = "crawlee"
-version = "1.1.1b8"
+version = "1.1.1b9"
 source = { editable = "." }
 dependencies = [
     { name = "cachetools" },

{crawlee-1.1.1b8 → crawlee-1.1.1b9}/website/package.json

@@ -18,6 +18,7 @@
     "devDependencies": {
         "@apify/eslint-config-ts": "^0.4.0",
         "@apify/tsconfig": "^0.1.0",
+        "@apify/ui-icons": "^1.23.0",
         "@docusaurus/module-type-aliases": "3.9.1",
         "@docusaurus/types": "3.9.1",
         "@types/react": "^19.0.0",

crawlee-1.1.1b9/website/src/components/LLMButtons.jsx

@@ -0,0 +1,510 @@
+import {
+    AnthropicIcon,
+    ChatGptIcon,
+    CheckIcon,
+    ChevronDownIcon,
+    CopyIcon,
+    ExternalLinkIcon,
+    LoaderIcon,
+    MarkdownIcon,
+    PerplexityIcon,
+} from '@apify/ui-icons';
+import { useLocation } from '@docusaurus/router';
+import clsx from 'clsx';
+import React, {
+    useCallback,
+    useEffect,
+    useMemo,
+    useRef,
+    useState,
+} from 'react';
+
+import styles from './LLMButtons.module.css';
+
+const DROPDOWN_OPTIONS = [
+    {
+        label: 'Copy for LLM',
+        description: 'Copy page as Markdown for LLMs',
+        showExternalIcon: false,
+        icon: CopyIcon,
+        value: 'copyForLLM',
+        analytics: {
+            buttonText: 'Copy for LLM',
+            element: 'llm-buttons.copyForLLM',
+        },
+    },
+    {
+        label: 'View as Markdown',
+        description: 'View this page as plain text',
+        icon: MarkdownIcon,
+        value: 'viewAsMarkdown',
+        showExternalIcon: true,
+        analytics: {
+            buttonText: 'View as Markdown',
+            element: 'llm-buttons.viewAsMarkdown',
+        },
+    },
+    {
+        label: 'Open in ChatGPT',
+        description: 'Ask questions about this page',
+        icon: ChatGptIcon,
+        value: 'openInChatGPT',
+        showExternalIcon: true,
+        analytics: {
+            buttonText: 'Open in ChatGPT',
+            element: 'llm-buttons.openInChatGPT',
+        },
+    },
+    {
+        label: 'Open in Claude',
+        description: 'Ask questions about this page',
+        icon: AnthropicIcon,
+        value: 'openInClaude',
+        showExternalIcon: true,
+        analytics: {
+            buttonText: 'Open in Claude',
+            element: 'llm-buttons.openInClaude',
+        },
+    },
+    {
+        label: 'Open in Perplexity',
+        description: 'Ask questions about this page',
+        icon: PerplexityIcon,
+        value: 'openInPerplexity',
+        showExternalIcon: true,
+        analytics: {
+            buttonText: 'Open in Perplexity',
+            element: 'llm-buttons.openInPerplexity',
+        },
+    },
+];
+
+const CHAT_GPT_BASE = 'https://chatgpt.com/?hints=search&q=';
+const CLAUDE_BASE = 'https://claude.ai/new?q=';
+const PERPLEXITY_BASE = 'https://www.perplexity.ai/search/new?q=';
+
+const getPrompt = (currentUrl) => `Read from ${currentUrl} so I can ask questions about it.`;
+const getMarkdownUrl = (currentUrl) => {
+    const url = new URL(currentUrl);
+    url.pathname = `${url.pathname.replace(/\/$/, '')}.md`;
+    return url.toString();
+};
+
+const trackClick = (buttonText, element) => {
+    if (typeof window !== 'undefined' && window.analytics) {
+        window.analytics.track('Clicked', {
+            app: 'crawlee',
+            button_text: buttonText,
+            element,
+        });
+    }
+};
+
+const getOptionHref = (value, currentUrl) => {
+    if (!currentUrl) {
+        return undefined;
+    }
+
+    switch (value) {
+        case 'viewAsMarkdown':
+            return getMarkdownUrl(currentUrl);
+        case 'openInChatGPT':
+            return `${CHAT_GPT_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;
+        case 'openInClaude':
+            return `${CLAUDE_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;
+        case 'openInPerplexity':
+            return `${PERPLEXITY_BASE}${encodeURIComponent(getPrompt(currentUrl))}`;
+        default:
+            return undefined;
+    }
+};
+
+const Menu = ({
+    className,
+    components = {},
+    onMenuOpen,
+    onSelect,
+    options = [],
+}) => {
+    const [isOpen, setIsOpen] = useState(false);
+    const [focusedIndex, setFocusedIndex] = useState(0);
+    const menuRef = useRef(null);
+    const menuItemRefs = useRef([]);
+
+    const MenuBaseComponent = components.MenuBase;
+
+    const closeMenu = useCallback(() => {
+        setIsOpen(false);
+        setFocusedIndex(0);
+    }, []);
+
+    const toggleMenu = useCallback(() => {
+        setIsOpen((prev) => {
+            if (!prev) {
+                setFocusedIndex(0);
+            }
+            return !prev;
+        });
+    }, []);
+
+    const handleKeyDown = useCallback(
+        (event) => {
+            if (event.key === 'Enter' || event.key === ' ') {
+                event.preventDefault();
+                toggleMenu();
+            } else if (event.key === 'ArrowDown') {
+                event.preventDefault();
+                if (!isOpen) {
+                    toggleMenu();
+                } else {
+                    setFocusedIndex((prev) => (prev + 1) % options.length);
+                }
+            } else if (event.key === 'ArrowUp') {
+                event.preventDefault();
+                if (isOpen) {
+                    setFocusedIndex((prev) => (prev - 1 + options.length) % options.length);
+                }
+            }
+        },
+        [toggleMenu, isOpen, options.length],
+    );
+
+    const handleOptionSelect = useCallback(
+        (option, event) => {
+            onSelect?.(option, event);
+            closeMenu();
+        },
+        [closeMenu, onSelect],
+    );
+
+    const handleMenuItemKeyDown = useCallback(
+        (event, option, index) => {
+            if (event.key === 'Enter' || event.key === ' ') {
+                event.preventDefault();
+                event.currentTarget.click();
+                return;
+            }
+
+            if (event.key === 'ArrowDown') {
+                event.preventDefault();
+                setFocusedIndex((index + 1) % options.length);
+                return;
+            }
+
+            if (event.key === 'ArrowUp') {
+                event.preventDefault();
+                setFocusedIndex((index - 1 + options.length) % options.length);
+                return;
+            }
+
+            if (event.key === 'Escape') {
+                event.preventDefault();
+                closeMenu();
+            }
+        },
+        [options.length, closeMenu],
+    );
+
+    useEffect(() => {
+        onMenuOpen?.(isOpen);
+    }, [isOpen, onMenuOpen]);
+
+    useEffect(() => {
+        if (isOpen && menuItemRefs.current[focusedIndex]) {
+            menuItemRefs.current[focusedIndex].focus();
+        }
+    }, [isOpen, focusedIndex]);
+
+    useEffect(() => {
+        if (!isOpen) {
+            return undefined;
+        }
+
+        const handleClickOutside = (event) => {
+            if (!menuRef.current?.contains(event.target)) {
+                closeMenu();
+            }
+        };
+
+        const handleEscape = (event) => {
+            if (event.key === 'Escape') {
+                closeMenu();
+            }
+        };
+
+        document.addEventListener('mousedown', handleClickOutside);
+        document.addEventListener('keydown', handleEscape);
+
+        return () => {
+            document.removeEventListener('mousedown', handleClickOutside);
+            document.removeEventListener('keydown', handleEscape);
+        };
+    }, [closeMenu, isOpen]);
+
+    return (
+        <div className={clsx(styles.menu, className)} ref={menuRef}>
+            <MenuBaseComponent
+                onClick={toggleMenu}
+                onKeyDown={handleKeyDown}
+                aria-haspopup="menu"
+                aria-expanded={isOpen}
+                aria-controls="llm-menu"
+            />
+            {isOpen && (
+                <div className={styles.menuDropdown} role="menu" id="llm-menu">
+                    {options.map((option, index) => {
+                        const WrapperComponent = option.href ? 'a' : 'button';
+
+                        return (
+                            <WrapperComponent
+                                key={option.value}
+                                ref={(el) => {
+                                    menuItemRefs.current[index] = el;
+                                }}
+                                className={styles.menuOptionWrapper}
+                                role="menuitem"
+                                tabIndex={0}
+                                href={option.href}
+                                target={option.target}
+                                rel={option.rel}
+                                type={option.href ? undefined : 'button'}
+                                onClick={(event) => {
+                                    if (!option.href) {
+                                        event.preventDefault();
+                                    }
+                                    handleOptionSelect(option, event);
+                                }}
+                                onKeyDown={(e) => handleMenuItemKeyDown(e, option, index)}
+                            >
+                                <Option {...option} />
+                            </WrapperComponent>
+                        );
+                    })}
+                </div>
+            )}
+        </div>
+    );
+};
+
+function getButtonText({ status }) {
+    switch (status) {
+        case 'loading':
+            return 'Copying...';
+        case 'copied':
+            return 'Copied';
+        default:
+            return 'Copy for LLM';
+    }
+}
+
+const onCopyAsMarkdownClick = async ({ setCopyingStatus, currentUrl }) => {
+    const sourceUrl = currentUrl || (typeof window !== 'undefined' ? window.location.href : '');
+
+    if (!sourceUrl) {
+        return;
+    }
+
+    trackClick('Copy for LLM', 'llm-buttons.copyForLLM');
+
+    const markdownUrl = getMarkdownUrl(sourceUrl);
+
+    try {
+        setCopyingStatus('loading');
+
+        const response = await fetch(markdownUrl);
+
+        if (!response.ok) {
+            throw new Error(`Failed to fetch markdown: ${response.status}`);
+        }
+
+        const markdownContent = await response.text();
+        await navigator.clipboard.writeText(markdownContent);
+        setCopyingStatus('copied');
+    } catch (error) {
+        console.error('Failed to copy markdown content:', error);
+    } finally {
+        setTimeout(() => setCopyingStatus('idle'), 2000);
+    }
+};
+
+const COPYING_STATUS_ICON = {
+    loading: <LoaderIcon size={16} />,
+    copied: <CheckIcon size={16} />,
+    idle: <CopyIcon size={16} />,
+}
+
+const MenuBase = React.forwardRef(({
+    copyingStatus,
+    setCopyingStatus,
+    chevronIconRef,
+    currentUrl,
+    ...buttonProps
+}, ref) => {
+    const mergedButtonProps = {
+        ...buttonProps,
+        tabIndex: buttonProps.tabIndex ?? 0,
+    };
+
+    return (
+        <div className={styles.llmButtonWrapper}>
+            <div
+                ref={ref}
+                className={styles.llmButton}
+                {...mergedButtonProps}
+            >
+                <div
+                    className={styles.copyUpIconWrapper}
+                    onClick={(event) => {
+                        event.stopPropagation();
+                        onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });
+                    }}
+                >
+                    {COPYING_STATUS_ICON[copyingStatus]}
+                </div>
+                <span
+                    onClick={(event) => {
+                        event.stopPropagation();
+                        onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });
+                    }}
+                    className={styles.llmButtonText}
+                >
+                    {getButtonText({ status: copyingStatus })}
+                </span>
+                <div className={styles.chevronIconWrapper}>
+                    <ChevronDownIcon
+                        size="16"
+                        color="currentColor"
+                        className={styles.chevronIcon}
+                        ref={chevronIconRef}
+                    />
+                </div>
+            </div>
+        </div>
+    );
+});
+MenuBase.displayName = 'MenuBase';
+
+const Option = ({ label, description, showExternalIcon, icon }) => {
+    const Icon = icon ?? CopyIcon;
+
+    return (
+        <div className={styles.menuOption}>
+            <Icon size={16} className={styles.menuOptionIcon} />
+            <div className={styles.menuOptionText}>
+                <span className={styles.menuOptionLabel}>{label}</span>
+                <span className={styles.menuOptionDescription}>{description}</span>
+            </div>
+            {showExternalIcon && (
+                <ExternalLinkIcon
+                    size={16}
+                    className={styles.menuOptionExternalIcon}
+                />
+            )}
+        </div>
+    );
+};
+
+export default function LLMButtons() {
+    const [copyingStatus, setCopyingStatus] = useState('idle');
+    const [isMarkdownAvailable, setIsMarkdownAvailable] = useState(false);
+    const chevronIconRef = useRef(null);
+    const location = useLocation();
+
+    const currentUrl = typeof window !== 'undefined'
+        ? `${window.location.origin}${location.pathname}${location.search}${location.hash}`
+        : '';
+
+    useEffect(() => {
+        if (!currentUrl) {
+            // TODO: Feel free to tell me how to fix this 🤦‍♂️
+            // eslint-disable-next-line react-hooks/set-state-in-effect
+            setIsMarkdownAvailable(false);
+            return undefined;
+        }
+
+        const controller = new AbortController();
+        const markdownUrl = getMarkdownUrl(currentUrl);
+
+        const checkMarkdownAvailability = async () => {
+            try {
+                const response = await fetch(markdownUrl, {
+                    method: 'HEAD',
+                    signal: controller.signal,
+                });
+                setIsMarkdownAvailable(response.ok);
+            } catch (error) {
+                if (error.name === 'AbortError') {
+                    return;
+                }
+                setIsMarkdownAvailable(false);
+            }
+        };
+
+        checkMarkdownAvailability();
+
+        return () => {
+            controller.abort();
+        };
+    }, [currentUrl]);
+
+    const menuOptions = useMemo(
+        () => DROPDOWN_OPTIONS.map((option) => {
+            const href = getOptionHref(option.value, currentUrl);
+
+            if (option.value === 'viewAsMarkdown') {
+                if (!isMarkdownAvailable) {
+                    return null;
+                }
+            }
+
+            return {
+                ...option,
+                href,
+                target: href ? '_blank' : undefined,
+                rel: href ? 'noopener noreferrer' : undefined,
+            };
+        }).filter(Boolean),
+        [isMarkdownAvailable, currentUrl],
+    );
+
+    const onMenuOptionClick = useCallback(
+        (option, event) => {
+            if (!option) {
+                return;
+            }
+
+            if (option.analytics) {
+                trackClick(option.analytics.buttonText, option.analytics.element);
+            }
+
+            if (option.value === 'copyForLLM') {
+                event?.preventDefault();
+                onCopyAsMarkdownClick({ setCopyingStatus, currentUrl });
+            }
+        },
+        [setCopyingStatus, currentUrl],
+    );
+
+    return (
+        <Menu
+            className={styles.llmMenu}
+            onMenuOpen={(isOpen) => chevronIconRef.current?.classList.toggle(
+                styles.chevronIconOpen,
+                isOpen,
+            )}
+            components={{
+                MenuBase: (props) => (
+                    <MenuBase
+                        copyingStatus={copyingStatus}
+                        setCopyingStatus={setCopyingStatus}
+                        chevronIconRef={chevronIconRef}
+                        currentUrl={currentUrl}
+                        {...props}
+                    />
+                ),
+            }}
+            onSelect={onMenuOptionClick}
+            options={menuOptions}
+        />
+    );
+}

crawlee-1.1.1b9/website/src/components/LLMButtons.module.css

@@ -0,0 +1,151 @@
+.llmMenu {
+    display: flex;
+    justify-content: flex-end;
+    flex: 0 0 auto;
+  }
+  
+  @media (max-width: 996px) {
+    .llmMenu {
+      width: 100%;
+      justify-content: flex-start;
+    }
+  }
+  
+  .llmButtonWrapper {
+    display: flex;
+    justify-content: flex-end;
+    width: auto;
+  }
+  
+  .llmButton {
+    display: flex;
+    align-items: center;
+    border-radius: 0.5rem;
+    border: 1px solid var(--color-separator);
+    background-color: var(--color-background-subtle);
+    cursor: pointer;
+    transition: background-color 0.2s ease-in-out, border-color 0.2s ease-in-out;
+  }
+  
+  .copyUpIconWrapper {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    padding: 0.6rem 0.5rem 0.6rem 0.8rem;
+  }
+  
+  .llmButtonText {
+    display: flex;
+    align-items: center;
+    padding-right: 0.8rem;
+    border-right: 1px solid var(--color-separator);
+    margin: 0;
+    font: 400 0.875rem/1.4 Inter, sans-serif;
+  }
+  
+  .chevronIconWrapper {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    padding-inline: 0.25rem;
+  }
+  
+  .chevronIcon {
+    transition: transform 0.2s ease-in-out;
+  }
+  
+  .chevronIconOpen {
+    transform: rotate(180deg);
+  }
+  
+  .menu {
+    position: relative;
+    width: fit-content;
+  }
+  
+  .menuDropdown {
+    position: absolute;
+    right: 0;
+    margin-top: 0.5rem;
+    padding: 0.375rem;
+    border-radius: 0.75rem;
+    border: 1px solid var(--color-separator);
+    background-color: var(--color-background);
+    box-shadow: 0 12px 32px rgb(10 11 36 / 20%);
+    min-width: 17rem;
+    max-width: min(20rem, calc(100vw - 1.5rem));
+    z-index: 2;
+    display: flex;
+    flex-direction: column;
+    gap: 0.25rem;
+  }
+  
+  @media (max-width: 996px) {
+    .menuDropdown {
+      left: 0;
+      right: auto;
+      width: min(20rem, calc(100vw - 1.5rem));
+    }
+  }
+  
+  .menuOption {
+    display: flex;
+    gap: 0.5rem;
+    padding: 0.25rem 0.5rem;
+    border-radius: 0.5rem;
+    transition: background-color 0.15s ease-in-out;
+  }
+  
+  .menuOption:hover {
+    background: var(--color-hover);
+  }
+  
+  .menuOptionWrapper {
+    border: none;
+    background: transparent;
+    padding: 0;
+    text-align: left;
+    width: 100%;
+    display: block;
+    text-decoration: none;
+    color: inherit;
+    cursor: pointer;
+    outline: none;
+  }
+  
+  .menuOptionWrapper:focus-visible .menuOption {
+    background: var(--color-hover);
+    outline-offset: -2px;
+  }
+  
+  .menuOptionIcon,
+  .menuOptionExternalIcon {
+    flex-shrink: 0;
+  }
+  
+  .menuOptionIcon {
+    margin-top: 0.2rem;
+  }
+  
+  .menuOptionText {
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+    gap: 0.125rem;
+    line-height: 1rem;
+    padding: 4px 0;
+  }
+  
+  .menuOptionLabel {
+    margin: 0;
+    font-size: 0.875rem;
+    line-height: 1rem;
+    font-weight: 400;
+    color: var(--ifm-font-color-base);
+  }
+  
+  .menuOptionDescription {
+    margin: 0;
+    font-size: 0.8rem;
+    color: var(--color-text-subtle);
+  }