PL jT UdZddlZddlZddlZddlZddlZddlmZmZm Z m Z m Z ddl Z e rddl mZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z m!Z!m"Z"ddl#m$Z$m%Z%ddl&m'Z'dZ(e e e)d <dZ*e e e)d <dZ+e e e)d <dZ,e e e)d <dZ-e e e)d <ddl.m/Z/m0Z0m1Z1ddl2m3Z3ddl4m5Z5m6Z7m8Z8ddl9m:Z:m;Z;ddlm?Z?ddl@Z@ejAeBZCdeDdeEfdZFdeGfdZHdeDfdZIdeDfdZJdeDfdZKdeDdeDfdZLdeDdeEfdZMdeEfdZNdeOeDfd ZPd!ZQd"e deEfd#ZRdd$e eDdeSe e e eDeeDe fffd%ZTde eDfd&ZUe3d'd()ZVd*d*deQfd+eDd,eDd-eDd$e eDd.eWde eDf d/ZX dd+eDd2eDd$e eDd3eWd4eEd5eDde eDfd6ZYd+eDd2eDd$e eDd7eWd8eWde eDf d9ZZd:eDdeDfd;Z[dd=eDd>eWdeDfd?Z\dd@deQfdAeeDdBeDdCeEd$e eDd.eWdeDf dDZ]ddEd@deQfd,eDdFeDdGeDdCeEd$e eDd.eWdeDfdHZ^deEfdIZ_deEfdJZ`eBdKkr eadLeadMe_ZbeZceEejddNd*eZfeEejddOd*eZge`ZheUZiebr!eIZjeadPejejdQkr eadRnejdSkr eadTnejdUkr eadVnejdWkr0eadXejddYd*enejdZkr ead[nejd\kr ead]negrBead^ejddOekd_nTefr ead`nFecreadaen-eadbn!eadceaddeehs"eadeeadfeadgneadheiebse@jldieadjehreadkeieadleQdmeVjmr0eadneVjneadoeVjodpeVjndqn eadreadseadteaduead*eadveadwead*eadxeadyeadzead{ead|ehread}ead~eadead*eadeadeadeadeadeadeadead*eadeadeadeadeadeadeadeadeadeadeadeadddlpmqZqmrZrdddddddddidd for Nous Subscribers) - Parallel: https://docs.parallel.ai (search, extract) - Tavily: https://tavily.com (search, extract, crawl) LLM Processing: - Uses OpenRouter API with Gemini 3 Flash Preview for intelligent content extraction - Extracts key excerpts and creates markdown summaries to reduce token usage Debug Mode: - Set WEB_TOOLS_DEBUG=true to enable detailed logging - Creates web_tools_debug_UUID.json in ./logs directory - Captures all tool calls, results, and compression metrics Usage: from web_tools import web_search_tool, web_extract_tool, web_crawl_tool # Search the web results = web_search_tool("Python machine learning libraries", limit=3) # Extract content from URLs content = web_extract_tool(["https://example.com"], format="markdown") # Crawl a website crawl_data = web_crawl_tool("example.com", "Find contact information") N)ListDictAnyOptional TYPE_CHECKING) Firecrawl)r_FirecrawlProxy_FIRECRAWL_CLS_CACHE_extract_scrape_payload_extract_web_search_results_firecrawl_backend_help_suffix_get_direct_firecrawl_config_get_firecrawl_client_get_firecrawl_gateway_url_has_direct_firecrawl_config_is_tool_gateway_ready_load_firecrawl_cls_normalize_result_list&_raise_web_backend_configuration_error_to_plain_objectcheck_firecrawl_api_key)_normalize_tavily_documents _normalize_tavily_search_results_tavily_request)_get_async_parallel_client_get_parallel_client)_get_exa_client_firecrawl_client_firecrawl_client_config_parallel_client_async_parallel_client _exa_client)async_call_llmextract_content_or_reasoningget_async_text_auxiliary_client) DebugSession)build_vendor_gateway_urlread_nous_access_tokenresolve_managed_tool_gateway)managed_nous_tools_enabledprefers_gateway) is_safe_url)check_website_accessnamereturncptj|}t|o|SN)osgetenvboolstrip)r.vals 3/home/kuhnn/.hermes/hermes-agent/tools/web_tools.py_has_envr8{s* )D//C # $ $$c~ ddlm}|diS#ttf$ricYSwxYw)z5Load the ``web:`` section from ~/.hermes/config.yaml.r load_configweb)hermes_cli.configr<get ImportError Exceptionr;s r7_load_web_configrBs^111111{}}  +++  # s #&<<c tdpd}|dvr|Sdt dpt dp t fdt dfd t d fd t d fd t dfdt dfdt ff}|D] \}}|r|cS dS)zDetermine which web backend to use (shared fallback). Reads ``web.backend`` from config.yaml (set by ``hermes tools``). Falls back to whichever API key is present for users who configured keys manually without running setup. backend>exaddgstavilysearxngparallel firecrawl brave-freerKFIRECRAWL_API_KEYFIRECRAWL_API_URLrJPARALLEL_API_KEYrHTAVILY_API_KEYrF EXA_API_KEYrI SEARXNG_URLrLBRAVE_SEARCH_API_KEYrG)rBr?lowerr5r8r_ddgs_package_importable) configuredbackend_candidatesrD availables r7 _get_backendrYs#$$((339r@@BBHHJJJ``` h233px@S7T7TpXnXpXpq X0112 8,--. ''( H]++, x 6778 )++,1  NNN  ;r9c tdS)ujDetermine which backend to use for web_search specifically. Selection priority: 1. ``web.search_backend`` (per-capability override) 2. ``web.backend`` (shared fallback — existing behavior) 3. Auto-detect from env vars This enables using different providers for search vs extract (e.g. SearXNG for search + Firecrawl for extract). search_get_capability_backendr9r7_get_search_backendr_s #8 , ,,r9c tdS)uDetermine which backend to use for web_extract specifically. Selection priority: 1. ``web.extract_backend`` (per-capability override) 2. ``web.backend`` (shared fallback — existing behavior) 3. Auto-detect from env vars extractr\r^r9r7_get_extract_backendrbs #9 - --r9 capabilityct}||dpd}|rt |r|St S)zShared helper for per-capability backend selection. Reads ``web.{capability}_backend`` from config; if set and available, uses it. Otherwise falls through to the shared ``_get_backend()``. _backendrE)rBr?rTr5_is_backend_availablerY)rccfgspecifics r7r]r]sm   C:///006B==??EEGGH)(33 >>r9rDc(|dkrtdS|dkrtdS|dkrtS|dkrtdS|dkrtd S|d krtd S|d krtSd S)z:Return True when the selected backend is currently usable.rFrQrJrOrKrHrPrIrRrLrSrGF)r8rrU)rDs r7rfrfs% &&&**++++&((((()))) &&&,.///&'))) 5r9c2 ddl}dS#t$rYdSwxYw)aJReturn True when the ``ddgs`` Python package can be imported. ddgs is the only backend whose availability is driven by a package presence rather than an env var / config entry. Wrapped in a helper so auto-detect and ``_is_backend_available`` share the same check (and tests can monkeypatch a single symbol). rNTF)rGr@)rGs r7rUrUs7 t uus  c gdS)uReturn tool metadata env vars for the currently enabled web backends. The gateway env vars are always reported — they're metadata strings used by the tool registry to light up the tool when the variable is set. Gating them on ``managed_nous_tools_enabled()`` only saved string noise in the metadata list, but cost a synchronous HTTP refresh against the Nous portal on every CLI startup (invoked at tool-registration time). The behavioral contract is: if the env var is set, the tool sees it; if not, it doesn't. Not-logged-in users simply don't have the vars set, so the extra entries are harmless. ) rQrOrPrMrNFIRECRAWL_GATEWAY_URLTOOL_GATEWAY_DOMAINTOOL_GATEWAY_SCHEMETOOL_GATEWAY_USER_TOKENr^r^r9r7_web_requires_envrps    r9clientcddlm}tt|ddpd}||jpd}|dkp|dS)z?Return True when the resolved auxiliary backend is Nous Portal.r)urlparsebase_urlrEznousresearch.comz.nousresearch.com) urllib.parsertstrgetattrhostnamerTendswith)rrrtruhosts r7_is_nous_auxiliary_clientr| ss%%%%%%76:r228b99H HX   ' -2 4 4 6 6D % % K7J)K)KKr9modelctd\}}tjdd}|p|p|}i}|1t |r"ddlm}ddlm}|p d|i}|||fS) zHResolve the current web-extract auxiliary client, model, and extra body. web_extractAUXILIARY_WEB_EXTRACT_MODELrENr)get_auxiliary_extra_body)nous_portal_tagstags) r%r2r3r5r|agent.auxiliary_clientragent.portal_tagsr)r}rr default_modelconfigured_modeleffective_model extra_bodyrrs r7_resolve_web_extract_auxiliaryr)s;MJJFMy!>CCIIKK@/@=O!#J 7??CCCCCC666666--//OF500k chars), uses chunked processing with synthesis. For extremely large content (>2M chars), refuses to process entirely. Args: content (str): The raw content to process url (str): The source URL (for context, optional) title (str): The page title (for context, optional) model (str): The model to use for processing (default: google/gemini-3-flash-preview) min_length (int): Minimum content length to trigger processing (default: 5000) Returns: Optional[str]: Processed markdown content, or None if content too short or processing fails ii 順rqi@Bz 2MB limit). Refusing to process.z[Content too large to process: z.1fzdMB. Try using web_crawl with specific extraction instructions, or search for a more focused source.]z:Content too short (%d < %d chars), skipping LLM processingNzTitle: zSource:   rEz5Content large (%d chars). Using chunked processing...z+Processing content with LLM (%d characters)4 [... summary truncated for context management ...]r?z*Content processed: %d -> %d chars (%.1f%%)dzweb_extract LLM summarization failed (%s). Tip: increase auxiliary.web_extract.timeout in config.yaml or switch to a faster auxiliary model.xu' [Content truncated — showing first , of z chars. LLM summarization timed out. To fix: increase auxiliary.web_extract.timeout in config.yaml, or use a faster auxiliary model. Use browser_navigate for the full page.]) lenloggerwarningdebugappendjoininfo_process_large_content_chunked_call_summarizer_llmrArw)rrrr}rMAX_CONTENT_SIZECHUNK_THRESHOLD CHUNK_SIZEMAX_OUTPUT_SIZE content_lensize_mb context_info context_strprocessed_contentprocessed_lengthcompression_ratioe truncateds r7process_content_with_llmr@s 8!OJO@'ll  ) ) )!I-G NNY[b c c cgWgggg g  # # LLUWbdn o o o4  3    1% 1 1 2 2 2  2    03 0 0 1 1 1:FNdii --66B   ( ( KKOQ\ ] ] ]7eZ   A;OOO"6w U"S"SSSSSSS  ~$%%77$56F6F$GKC%C! ##455 BMPQ// 0; > >WZ  KKDkScevy|e| } } }   5 FF4C4L    ,_,- w< %d charszChunk %d/%d failed: %s2)rrrrrArrw)rrrsummaryrchunksrr}s r7summarize_chunkz7_process_large_content_chunked..summarize_chunk8sL #Oi!mOOVOOOJ0 % G D DiRSmUXY_U`U`befsbtbtvy{BwCwCDDDg% % # # # NN3Y]CKKQTUVQWQWX[Y[X[Q\ ] ] ]d? " " " " " " #sBB C!ACC!C!c.g|]\}}||Sr^r^).0ichunkrs r7 z2_process_large_content_chunked..Ls) I I I81e__Q & & I I Ir9return_exceptionsTNz#Chunk summarization task failed: %sc|dS)Nrr^)xs r7z0_process_large_content_chunked..Zs qtr9)keyz ## Section rrzAll chunk summarizations failedzB[Failed to process large content: all chunk summarizations failed]zGot %d/%d chunk summariesz [... truncated ...]zSynthesizing %d summaries...z --- a'You have been given summaries of different sections of a large document. Synthesize these into ONE cohesive, comprehensive summary that: 1. Removes redundancy between sections 2. Preserves all key facts, figures, and actionable information 3. Is well-organized with clear structure 4. Is under z characters zSECTION SUMMARIES: z, Create a single, unified markdown summary.z9No auxiliary model for synthesis, concatenating summariesrrrzdYou synthesize multiple summaries into one cohesive, comprehensive summary. Be thorough but concise.rrrrrrz3Synthesis LLM returned empty content, retrying onceu>Synthesis failed after retry — concatenating chunk summariesrrz+Synthesis complete: %d -> %d chars (%.2f%%)rzSynthesis failed: %sz. [... truncated due to synthesis failure ...]r^)rrrrrintrwtupler enumeratergather isinstance BaseExceptionrsortedrrrr#r$rA)rrr}rrrrtasksresultssuccessful_results result_item summariesrrresultcombined_summariessynthesis_promptrrrfallbackrr final_summary original_len final_len compressionrrrs `` @@r7rrsr,F 1c'llJ / /!j.() e KK8#f++zRRR##S#U3PXY\P]K]E^########( J I I Iy7H7H I I IENEBTBBBBBBBBG// k= 1 1  NN@+ N N N !!+....I$%7^^LLLGG 7  G   E9q=EEGEE F F F T 6777SS KK+S^^S[[III 9~~1 v;; ( (,_,-0IIF  KK.I???&++I66 .  . .  . . . . .72PQV2W2W/ OZ  _  NNV W W W{{9--H8}}..#$4_$458QQO"$!/UVV,<==     3(2K %'66+666666664X>>  C NNP Q Q Q+::k::::::::H8BBM  NN[ \ \ \{{9--H8}}..#$4_$458QQO }   / /)*:?*:;>vvM7||  && 2>2B2Bi,..  A   <mU;;; ;         &''18B$$W---d  8#;#;#=#= 2133H   8MM KK5 ue   %OOE599M+.}/@/@/L/L/P/PQVXZ/[/[+\+\(jquMMM 14[1A1A-.)?;;;  %%%4CFF44  T9%%%#, )?;;; )$$$$$$$$%s2(( F6DF HA4HHHTurlsformatuse_llm_processingc T #$Kddlm}ddlm}|D]N}||s|||rt jdddcSO||||ddddddggd } td t|g} g} |D]A} t| s| | d d d d ,| | B| sg} n#t} ddl m}m}| r || nd}||s\|5|s!t jd|jdddS|}|t jddddStd|jt| ddl}||jr|| |d{V} n"t-j|j| |d{V} | r| | z} d| i}t|dg}td|||d<tt j||d<|p t3#t5}|rq|rntd|d d#fd$|dg}$fd|D}t-j|ddid{V}|D]}t9|t:rtd|3|\}}}|d d!} |d"krG|d# ||d$xxd%z cc<td&| |d'kr7|d# |td(| td)| n|r7|s5td*|d d+|dgD]W}|d d!} t|d,d }td-| |Xd.|dgD}d|i}|dgkrt?d/}tA|} n&t j|d0d1}tA|} t| |d2<|d d3tB"d4|tB#| S#tH$r}!d5tK|!}"t&d6|"|"|d7<tB"d4|tB#t?|"cYd}!~!Sd}!~!wwxYw)8a# Extract content from specific web pages using available extraction API backend. This function provides a generic interface for web content extraction that can work with multiple backends. Currently uses Firecrawl. Args: urls (List[str]): List of URLs to extract content from format (str): Desired output format ("markdown" or "html", optional) use_llm_processing (bool): Whether to process content with LLM for summarization (default: True) model (Optional[str]): The model to use for LLM processing (defaults to current auxiliary backend model) min_length (int): Minimum content length to trigger LLM processing (default: 5000) Security: URLs are checked for embedded secrets before fetching. Returns: str: JSON string containing extracted content. If LLM processing is enabled and successful, the 'content' field will contain the processed markdown summary instead of raw content. Raises: Exception: If extraction fails or API key is not set r) _PREFIX_RE)unquoteFz_Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs.r)r2r3r4r}rN)r rpages_extractedpages_processed_with_llmrrcompression_metricsprocessing_appliedz!Extracting content from %d URL(s)rE:Blocked: URL targets a private or internal network addressrrrr)get_active_extract_providerrzy is a search-only backend and cannot extract URL content. Set web.extract_backend to firecrawl, tavily, exa, or parallel.rzcNo web extract provider configured. Set web.extract_backend to firecrawl, tavily, exa, or parallel.zWeb extract via %s: %d URL(s))r3rzExtracted content from %d pagesr8rz3Processing extracted content with LLM (parallel)...r;llm_processingcK|dd}|dd}|ddp|dd}|s|ddfSt|}t||| d{V}|r3t|}|d kr||z nd }||d<||d<|||| d }||d fS|||d dd d}||dfS)zHProcess a single result with LLM and return updated result with metrics.r Unknown URLrrE raw_contentrN no_contentrrr original_sizeprocessed_sizer model_used processedcontent_too_shortrrFrGrrHreason too_shortr?rr) rrrrCrFrIrGrmetricsrrs r7process_single_resultz/web_extract_tool..process_single_resultsRjj 66 7B//$jj;;Xvzz)UW?X?X "6!455 #K 0 0 #;e_j## 8%(^^NJWZ[J[J[(F(Fad%)2F9%,7F=) #)6*8->&5 G"7K77 #)6*7-0&*"5 G"7K77r9c&g|] }|Sr^r^)rrrPs r7rz$web_extract_tool..s%NNNv**622NNNr9rTz%Web result processing task failed: %srrBrIr:r9rz%s (processed)rMz&%s (no processing - content too short)z%s (no content to process)PLLM processing requested but no auxiliary model available, returning raw contentllm_processing_unavailablerCz%s (%d characters)cg|]h}|dd|dd|dd|ddd|vr d|diniiSrrErrrr=blocked_by_policyr?rrs r7rz$web_extract_tool..9s     uuUB''w++55B//w    GZ]^F^F^)1-@+ABBdf      r9z%Content was inaccessible or not foundrrrbase64_image_removalweb_extract_toolzError extracting content: rr)' agent.redactr6rvr7r[r'r(rrrr,rrbr%r>rsupports_extract display_namer.inspectiscoroutinefunctionrar to_threadr?rcheck_auxiliary_modelrrrrr$rr)r*r+rArwr)%r2r3r4r}rr6r7_urlr, safe_urls ssrf_blockedrrrDr>r-r.r_rr8auxiliary_available results_listrprocessed_resultsrrrOstatuscontent_lengthtrimmed_resultstrimmed_responser0cleaned_resultrr1rrPs% ` @@r7r[r[Js>('''''$$$$$$   T " " j&7&7 &F&F : =    "4$   $%"# ! O"\% 7TCCC -/  & &Cs## &##Y%%   %%%%B GG*,,G         6=F((111$Hx'@'@'B'B'0I0I0K0K':',#+#8!<!<!<&+    7688#:',!<&+     KK/I    NNN**8+;<<  ( 0 06 0 J JJJJJJJ!( 1$i!!!  -"W,Gw'hll9b99:: 5GGG-<)*47 88L8L4M4M01B#@#B#B355 O G"5O G KKM N N N 0 1 8 89I J J J) 8) 8) 8) 8) 8) 8X$<< 266LNNNNNNNE'.ne&Tt&T&T T T T T T T  1 F F k=99NN#JKXXX*5'jj 66[((#$9:AA'JJJ#$>???1D???KK 0#6666{**#$9:AA'JJJKK H#NNNNNN#?EEEE F " [*= [qrrr 45<<=YZZZ",,y"55 G Gjj 66!$VZZ r%B%B!C!C 0#~FFFF   \\)R00    &7    * *b 0 0$%LMMK0==NN*%5aeTTTK0==N14^1D1D-.,-445KLLL *O<<<  %%%9Q99  T9%%%#, *O<<< )$$$$$$$$%s-5C'V$VPV X'(A4X"X'"X'basic instructionsdepthc K|||||ddddddggd} |p tt}t}ddlm} m} |r | |nd} | K| s7| s!tj d| j ddd Sd} | | } | =| s)tj dd td dd S| | d sd |}t|stj d|ddddgid St|} | rct d| d| dtj d|dd| d| d| d| dddgid Sddlm} | rt)ddSt d| j|ddl}|dd}|r||d<|| jr| j|fi|d{V}nt3j| j|fi|d{V}t7|t8sdgi}|dgt=|dg}t d |||d!<t=tj ||d"<|r|rt d#|d$ d%fd&fd'|dgD}t3j!|d(d)id{V}|D]j}t7|tDrt #d*|3|\}}}|d+kr+|d, ||d-xxd.z cc<k|r7|s5t #d/|d$ d0d1|dgD}tj d|id2d3}tI|}t=||d4<tJ&d5|tJ'|Stj dd6td7dd S#tP$r}d8tS|}t *d9|||d:<tJ&d5|tJ't)|cYd}~Sd}~wwxYw);a Crawl a website with specific instructions using available crawling API backend. This function provides a generic interface for web crawling that can work with multiple backends. Currently uses Firecrawl. Args: url (str): The base URL to crawl (can include or exclude https://) instructions (str): Instructions for what to crawl/extract using LLM intelligence (optional) depth (str): Depth of extraction ("basic" or "advanced", default: "basic") use_llm_processing (bool): Whether to process content with LLM for summarization (default: True) model (Optional[str]): The model to use for LLM processing (defaults to current auxiliary backend model) min_length (int): Minimum content length to trigger LLM processing (default: 5000) Returns: str: JSON string containing crawled content. If LLM processing is enabled and successful, the 'content' field will contain the processed markdown summary instead of raw content. Each page is processed individually. Raises: Exception: If crawling fails or API key is not set )rrorpr4r}rNr)r r pages_crawledr9rrr:r;)get_active_crawl_providerrFzo is a search-only backend and cannot crawl URLs. Set FIRECRAWL_API_KEY for crawling, or use web_search instead.rr?zFweb_crawl requires Firecrawl. Set FIRECRAWL_API_KEY, FIRECRAWL_API_URLz*, or use web_search + web_extract instead.)zhttp://https://rtrrEr<r=z#Blocked web_crawl for %s by rule %sr{rulemessagesource)r{rurw)rrrrrVrrrzWeb crawl via %s: %s)rpr rozCrawled %d pagesrrrz1Processing crawled content with LLM (parallel)...r;r@czK|dd}|dd}|dd}|s|ddfSt|}t|||d{V}|r8||d<||d<||t||rt||z ndd }||d fS|||ddd d }||d fS)NrrBrrErrDrCrrErIrJrKrMrN) rpage_urlrrrFrIrOrrs r7_process_tavily_crawlz-web_crawl_tool.._process_tavily_crawls%zz%??H"JJw33E$jjB77G":%t\99$'LLM&>wRWYhjt&u&u u u u u u uI <07}-,5y)*2]fijsftftZg8pI8V8VmpAP#Q#Q%w ;;&.bo47tWjllG!7K77r9c&g|] }|Sr^r^)rrYr{s r7rz"web_crawl_tool..s%WWWa..q11WWWr9rTz'Tavily crawl processing task failed: %srIr:r9rrRrScg|]h}|dd|dd|dd|ddd|vr d|diniiSrUrWrXs r7rz"web_crawl_tool..'sOOOmn()uuUB'7'7!%%QSBTBTabafafgprtauauABAFAFGNAOAO i iFY]^F^F^)1-@+ABBdf iOOOr9rrrweb_crawl_toolz`web_crawl has no available backend. Set FIRECRAWL_API_KEY (or FIRECRAWL_API_URL for self-hosted)zW, or set TAVILY_API_KEY for Tavily. Alternatively use web_search + web_extract instead.zError crawling website: rr)+rrbrYr%rsrsupports_crawlr]r'r(r^ is_availabler startswithr,r-rrr#rr$r.r_r`crawlrrardict setdefaultrr?rrrrrr)r*r+rArwr)rrorpr4r}rr,rfrDrsr-crawl_providerblocked_is_intr_ crawl_kwargsrrrrrhrrrOrirkr0rmrr1r{rs ` @@r7r~r~csj@("4$   $%"# ! O$v%B#@#B#B355..         8?H**7333D  %n.K.K.M.M %"2244 z#(-:222"'    "N  !6688N  %n.I.I.K.K %:$C,J,L,LCCC#      %>>"9:: '&&&s## rz9sRTVY0[0[/\#]kprrrr+3//G Q A76?T[\bTcdddz9sRTVahiras29&/7SY?fmnvfw)x)x0z0z/{#|KPQQQQ B A A A A Awyy @!-???? KK.0CS I I I NNN%*R88L +?@@ !5!5c!J!J\!J!JJJJJJJ!(!2"(#""1="" h-- +%r?    2 . . .  Y ; ;<EEgNNN'(BCCCqHCCC! [*= [qrrr 45<<=YZZZOOrzr~r~@IKMsNsNOOOO*i%A!Z_```K0==N585H5HO1 2 OO,o > > > KKMMM! ! z J#A#C#CJJJ       %%%7s1vv77  T9%%%#, (/::: )$$$$$$$$%sGB R>'A R>4A R>>A3R>2 R>KR>(R>> UA4U<UUctdd}|dvrt |St ddDS)z6Check whether the configured web backend is available.rDrE>rFrGrHrIrJrKrLc34K|]}t|VdSr1)rf)rrDs r7 z$check_web_api_key..Rs>  g&&r9)rFrJrKrHrIrLrG)rBr?rTr5rfany)rVs r7check_web_api_keyrMs}!##'' 266<<>>DDFFJ```$Z000 b  r9c.t\}}}|duS)zICheck if an auxiliary text model is available for LLM content processing.Nr)rrrs r7rbrbXs133LFAq  r9__main__u 🌐 Standalone Web Tools Modulez(========================================rMrNu✅ Web backend: rFz! Using Exa API (https://exa.ai)rJz+ Using Parallel API (https://parallel.ai)rHz( Using Tavily API (https://tavily.com)rIz Using SearXNG (search only): rRrLz- Using Brave Search free tier (search only)rGz2 Using DuckDuckGo via ddgs package (search only)z Using self-hosted Firecrawl: /z# Using direct Firecrawl cloud APIz! Using Firecrawl tool-gateway: z0 Firecrawl backend selected but not configuredu$❌ No web search backend configuredzWSet EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URLu;❌ No auxiliary model available for LLM content processingzVSet OPENROUTER_API_KEY, configure Nous Portal, or set OPENAI_BASE_URL + OPENAI_API_KEYuK⚠️ Without an auxiliary model, LLM content processing will be disabledu✅ Auxiliary model available: ru!🛠️ Web tools ready for use!u+🧠 LLM content processing available with z& Default min length for processing: z charsu&🐛 Debug mode ENABLED - Session ID: z Debug logs will be saved to: z/web_tools_debug_z.jsonu=🐛 Debug mode disabled (set WEB_TOOLS_DEBUG=true to enable)z Basic usage:zI from web_tools import web_search_tool, web_extract_tool, web_crawl_toolz import asyncioz # Search (synchronous)z/ results = web_search_tool('Python tutorials')z$ # Extract and crawl (asynchronous)z async def main():z? content = await web_extract_tool(['https://example.com'])zC crawl_data = await web_crawl_tool('example.com', 'Find docs')z asyncio.run(main())z LLM-enhanced usage:zC # Content automatically processed for pages >5000 chars (default)zA content = await web_extract_tool(['https://python.org/about/'])z# # Customize processing parametersz$ crawl_data = await web_crawl_tool(z 'docs.python.org',z 'Find key concepts',z, model='google/gemini-3-flash-preview',z min_length=3000z )z # Disable LLM processingzY raw_content = await web_extract_tool(['https://example.com'], use_llm_processing=False)z Debug mode:z # Enable debug loggingz export WEB_TOOLS_DEBUG=truez # Debug logs capture:z$ # - All tool calls with parametersz # - Original API responsesz # - LLM compression metricsz # - Final processed resultsz3 # Logs saved to: ./logs/web_tools_debug_UUID.jsonuL 📝 Run 'python test_web_tools_llm.py' to test LLM processing capabilities)registryr$ web_searchaSearch the web for information. Returns up to 5 results by default with titles, URLs, and descriptions. The query is passed through to the configured backend, so operators such as site:domain, filetype:pdf, intitle:word, -term, and "exact phrase" may work when the backend supports them.objectstringzThe search query to look up on the web. You may include backend-supported operators such as site:example.com, filetype:pdf, intitle:word, -term, or "exact phrase".)type descriptionintegerz3Maximum number of results to return. Defaults to 5.r)rrminimummaximumdefaultr )r propertiesrequired)r.rr ruExtract content from web page URLs. Returns page content in markdown format. Also works with PDF URLs (arxiv papers, documents, etc.) — pass the PDF link directly and it converts to markdown text. Pages under 5000 chars return full markdown; larger pages are LLM-summarized and capped at ~5000 chars per page. Pages over 2M chars are refused. If a URL fails or times out, use the browser tool to access it instead.arrayrz:List of URLs to extract content from (max 5 URLs per call))ritemsrmaxItemsr=c tt|dd|ddS)Nr rEr r)r )rr?argskws r7rrs1txx/D/DDHHU\^_L`L`aaar9u🔍r)r.toolsetschemahandlercheck_fn requires_envemojimax_result_size_charsc tt|dtr|dgddngdS)Nr2rmarkdown)r[rr?listrs r7rrsP/$.txx/?/?$F$FNRaR  BPZ \ \r9u📄) r.rrrrris_asyncrrr1)rFrE)r)v__doc__r'loggingr2rrtypingrrrrrhttpxrKrplugins.web.firecrawl.providerr r r r r rrrrrrrrrrplugins.web.tavily.providerrrrplugins.web.parallel.providerrrplugins.web.exa.providerrr__annotations__rr r!r"rr#r$r%tools.debug_helpersr&tools.managed_tool_gatewayr'r(_read_nous_access_tokenr)tools.tool_backend_helpersr*r+tools.url_safetyr,tools.website_policyr-sys getLogger__name__rrwr4r8rrBrYr_rbr]rfrUrrp$DEFAULT_MIN_LENGTH_FOR_SUMMARIZATIONr|rrrr)rrrrrrr[r~rrbprint web_availabletool_gateway_availabler3r5firecrawl_key_availablefirecrawl_url_availablenous_availabledefault_summarizer_modelrDrstripexitactive session_idlog_dirtools.registryrr$WEB_SEARCH_SCHEMAWEB_EXTRACT_SCHEMAregisterr^r9r7rs#'''R  ;;;;;;;;;;;;;; $######(544444 $(8C='''*.(3-..."&(3-&&&(, ,,,! Xc]!!! -,,,,, SRRRRRRR((((((555555  8 $ $ %3%4%%%%$c@ -S - - - -.c....      34& $    049L(,$LcLdLLLL / /(3- /5RUX`adXegkloqtltguIuCv / / / /x} k+< = = = : aa a a a C= a  a c] aaaaP ss ss C=s s  s  sc]sssslY YY C=Y Y  Y c] YYYYxccTg%g%3g%sg%3g%g%g%g%X#: V%V% s)V% V%V% C= V%  V%  V%V%V%V%v#: f%f% f%f% f% f% C= f%  f% f%f%f%f%T4t z E ,--- E(OOO&%''M3355"d929-@"#E#E#K#K#M#MNN"d929-@"#E#E#K#K#M#MNN**,,N<<>> ,.. +'++,,, e   E5 6 6 6 6  " " E? @ @ @ @  E< = = = =  ! ! E[YRY}b5Q5Q5W5W5Y5Y[[ \ \ \ \  $ $ EA B B B B    EF G G G G $ F EiYRY?R5S5S5Y5Y5[5[5b5bcf5g5gii j j j j $ F E7 8 8 8 8 # F ET6P6P6R6RTT U U U U ED E E E E 4555  2--// 2 2   L KLLL fggg [\\\\ J0HJJKKK   E -...e V