
    PL
jK              	      L   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z
mZmZmZmZ ddlmZmZmZmZ  ej        e          ZdadIdZ eh d	          Z eh d
          Z eh d           eh d           eh d           eh d           eh d          hZddddddZdJdZ ej        dej                    ej        dej                    ej        dej                    ej        dej                    ej        dej                    ej        dej                   gZ!dKd Z" ej#                    Z$da%d!e&d"<   d#a' e(            a)d$e&d%<   dLd'Z*dId(Z+ G d) d*e          Z,dMd.Z-dNd0Z.dOd1Z/dPd3Z0dQd6Z1dRd9Z2dSd<Z3dTdUdCZ4dVdFZ5dWdGZ6dXdHZ7dS )Yu  Entry point for the `computer_use` tool.

Universal (any-model) macOS desktop control via cua-driver's background
computer-use primitive. Replaces #4562's Anthropic-native `computer_20251124`
approach — the schema here is standard OpenAI function-calling so every
tool-capable model can drive it.

Return contract
---------------
For text-only results (wait, key, list_apps, focus_app, failures, etc.):
  JSON string.

For captures / actions with `capture_after=True`:
  A dict wrapped as the OpenAI-style multi-part tool-message content:

      {
        "_multimodal": True,
        "content": [
            {"type": "text", "text": "<human-readable summary + SOM index>"},
            {"type": "image_url",
             "image_url": {"url": "data:image/png;base64,<b64>"}},
        ],
        "text_summary": "<text used for fallback string content>",
      }

  run_agent.py's tool-message builder inspects `_multimodal` and emits a
  list-shaped `content` for OpenAI-compatible providers. The Anthropic
  adapter splices the base64 image into a `tool_result` block (see
  `agent/anthropic_adapter.py`). Every provider that supports multi-part
  tool content gets the image; text-only providers see the summary only.
    )annotationsN)AnyDictListOptionalTuple)ActionResultCaptureResultComputerUseBackend	UIElementreturnNonec                
    | a dS )a  Register a callback for computer_use approval prompts (used by CLI).

    Matches the terminal_tool._approval_callback pattern. The callback
    receives (action, args, summary) and returns one of:
      "approve_once" | "approve_session" | "always_approve" | "deny".
    N)_approval_callback)cbs    ;/home/kuhnn/.hermes/hermes-agent/tools/computer_use/tool.pyset_approval_callbackr   <   s         >   waitcapture	list_apps>
   keydragtypeclickscroll	focus_app	set_valueright_clickdouble_clickmiddle_click>   cmdshift	backspace>   r"   optionr$   >   qr"   ctrl>   r&   r"   r#   >   r&   r"   r#   r%   r"   r'   r%   )commandcontrolaltu   ⌘u   ⌥keysstr	frozensetc                v    d t          j        d|           D             }d |D             }t          |          S )Nc                    g | ]<}|                                 |                                                                 =S  )striplower.0ps     r   
<listcomp>z$_canon_key_combo.<locals>.<listcomp>^   s7    QQQ1qwwyyQQWWYY__QQQr   z\s*\+\s*c                D    g | ]}t                               ||          S r0   )_KEY_ALIASESgetr3   s     r   r6   z$_canon_key_combo.<locals>.<listcomp>_   s(    333\a##333r   )resplitr-   )r+   partss     r   _canon_key_combor=   ]   sB    QQd(C(CQQQE33U333EUr   zcurl\s+[^|]*\|\s*bashzcurl\s+[^|]*\|\s*shzwget\s+[^|]*\|\s*bashz\bsudo\s+rm\s+-[rf]z\brm\s+-rf\s+/\s*$z:\s*\(\)\s*\{\s*:\|:\s*&\s*\}textOptional[str]c                V    t           D ] }|                    |           r	|j        c S !d S )N)_BLOCKED_TYPE_PATTERNSsearchpattern)r>   pats     r   _is_blocked_typerE   n   s=    %  ::d 	;	4r   zOptional[ComputerUseBackend]_backendFset_always_allowr   c                 n   t           5  t          t          j                            dd                                          } | dv rddlm}  |            an'| dk    rt                      ant          d|           t          
                                 t          cd d d            S # 1 swxY w Y   d S )NHERMES_COMPUTER_USE_BACKENDcua>   
cua-driver rK   r   )CuaDriverBackendnoopz$Unknown HERMES_COMPUTER_USE_BACKEND=)_backend_lockrF   osenvironr9   r2   tools.computer_use.cua_backendrN   _NoopBackendRuntimeErrorstart)backend_namerN   s     r   _get_backendrX      s   	  :>>*GOOUUWWL888KKKKKK++--'''>>"#Z,#Z#Z[[[NN                 s   BB**B.1B.c                     t           5  t          +	 t                                           n# t          $ r Y nw xY wdaddd           n# 1 swxY w Y   dat                      adS )u-   Test helper — tear down the cached backend.NF)rP   rF   stop	Exception_session_auto_approverG   rH   r0   r   r   reset_backend_for_testsr]      s     
                    "EEMMMs,   A	+A	
8A	8A		AAc                  v    e Zd ZdZddZddZddZddZd d!dZd"dZ	d"dZ
d"dZd#dZd$dZd%dZd&d'dZd
S )(rT   z5Test/CI stub. Records calls; returns trivial results.r   r   c                "    g | _         d| _        d S NF)calls_startedselfs    r   __init__z_NoopBackend.__init__   s    79
r   c                    d| _         d S NTrb   rc   s    r   rV   z_NoopBackend.start   s    TT]]]r   c                    d| _         d S r`   rh   rc   s    r   rZ   z_NoopBackend.stop   s    EDMMMr   boolc                    dS rg   r0   rc   s    r   is_availablez_NoopBackend.is_available   s    44r   somNmoder,   appr?   r
   c           	     p    | j                             d||df           t          |ddd g |pdd          S )Nr   rn   ro   i   i   rM   )rn   widthheightpng_b64elementsro   window_title)ra   appendr
   )rd   rn   ro   s      r   r   z_NoopBackend.capture   sS    
9tC&@&@ABBB$d3&(ciRbJ J J 	Jr   r	   c                \    | j                             d|f           t          dd          S )Nr   Tokactionra   rw   r	   rd   kws     r   r   z_NoopBackend.click   s/    
7B-(((tG4444r   c                \    | j                             d|f           t          dd          S )Nr   Try   r|   r}   s     r   r   z_NoopBackend.drag   s/    
62,'''tF3333r   c                \    | j                             d|f           t          dd          S )Nr   Try   r|   r}   s     r   r   z_NoopBackend.scroll   s/    
8R.)))tH5555r   r>   c                `    | j                             dd|if           t          dd          S )Nr   r>   Try   r|   )rd   r>   s     r   	type_textz_NoopBackend.type_text   s4    
6FD>2333tF3333r   r+   c                `    | j                             dd|if           t          dd          S )Nr   r+   Try   r|   )rd   r+   s     r   r   z_NoopBackend.key   s4    
564.1222tE2222r   List[Dict[str, Any]]c                >    | j                             di f           g S )Nr   )ra   rw   rc   s    r   r   z_NoopBackend.list_apps   s"    
;+,,,	r   Fraise_windowc                b    | j                             d||df           t          dd          S )Nr   )ro   raiseTry   r|   )rd   ro   r   s      r   r   z_NoopBackend.focus_app   s8    
;l(K(KLMMMtK8888r   r   r   r   rj   )rm   N)rn   r,   ro   r?   r   r
   )r   r	   )r>   r,   r   r	   )r+   r,   r   r	   )r   r   )F)ro   r,   r   rj   r   r	   )__name__
__module____qualname____doc__re   rV   rZ   rl   r   r   r   r   r   r   r   r   r0   r   r   rT   rT      s       ??    21111111////J J J J J
5 5 5 54 4 4 46 6 6 64 4 4 43 3 3 3   9 9 9 9 9 9 9r   rT   argsDict[str, Any]r   c                   |                      d          pd                                                                }|st          j        ddi          S |dk    rA|                      dd          }t          |          }|rt          j        d|dd	          S |d
k    r|                      dd          }t          |          }t          D ]`}|                    |          rIt          |          t          |          k    r)t          j        dt          |           dd	          c S a|t          v rt          ||           }||S 	 t                      }	n1# t          $ r$}
t          j        d|
 dd	          cY d}
~
S d}
~
ww xY w	 t          |	||           S # t          $ r@}
t                               d|           t          j        d| d|
 i          cY d}
~
S d}
~
ww xY w)u   Main entry point — dispatched by tools.registry.

    Returns either a JSON string (text-only) or a dict marked `_multimodal`
    (image + summary) which run_agent.py wraps into the tool message.
    r{   rM   errorzmissing `action`r   r>   zblocked pattern in type text: z:Dangerous shell patterns cannot be typed via computer_use.)r   hintr   r+   zblocked key combo: z.Destructive system shortcuts are hard-blocked.Nz"computer_use backend unavailable: zARun `hermes tools` and enable Computer Use to install cua-driver.zcomputer_use %s failedz	 failed: )r9   r1   r2   jsondumpsrE   r=   _BLOCKED_KEY_COMBOSissubsetlensorted_DESTRUCTIVE_ACTIONS_request_approvalrX   r[   	_dispatchlogger	exception)r   kwargsr{   r>   rD   r+   comboblockederrbackendes              r   handle_computer_user      sn    hhx  &B--//5577F 9z7$67888 xx##t$$ 	:A#AAT    
 xx## &&* 	 	G&& 3w<<3u::+E+EzD6'??DDL# #      %%%--?J..   z=!==W
 
   	 	 	 	 	 	>&$/// > > >16:::z7v$;$;$;$;<========>s<   E 
F	%F>F	F	F 
G((5G#G(#G(r{   c                   t           rdS | t          v rdS t          }|dS t          | |          }	  || ||          }n4# t          $ r'}t
                              d|           d}Y d}~nd}~ww xY w|dk    rdS |dk    s|dk    r$t                              |            |dk    rda dS t          j	        d| d	          S )
z:Return None if approved, or a JSON error string if denied.Nzapproval callback failed: %sdenyapprove_onceapprove_sessionalways_approveTzdenied by user)r   r{   )
r\   rH   r   _summarize_actionr[   r   warningaddr   r   )r{   r   r   summaryverdictr   s         r   r   r     s     tt	B	z t--G"VT7++   5q999 .  t###w2B'B'B&!!!&&&$(!t: 0FCCDDDs   ? 
A0	A++A0c                   | dv rO|                     d          |  d|d          S |                     d          }|r|  dt          |           S | S | dk    r\|                     d          p|                     d          }|                     d	          p|                     d
          }d| d| S | dk    r0d|                     dd           d|                     dd           S | dk    r;|                     dd          }d|d d         t          |          dk    rdndz   S | dk    rd|                     dd          S | dk    r3d|                     dd          |                     d           rd!ndz   S | S )"N>   r   r   r    r!   elementz
 element #
coordinatez at r   from_elementfrom_coordinate
to_elementto_coordinatezdrag u    → r   zscroll 	direction?z xamount   r   r>   rM   ztype <   z...r   zkey r+   r   zfocus ro   r   z (raise))r9   tupler   )r{   r   coordsrcdstr>   s         r   r   r      s   III88I*99Y999&& 	100%,,000hh~&&E$((3D*E*Ehh|$$A(A(A&s&&&&&N+s33NNtxx!7L7LNNNxx##$tCRCy$$TRRHH.dhhvr**.../++//.AYAY3a::_abbMr   r   c           
     D
   t          |                    d                    }|dk    rzt          |                    dd                    }|dvrt          j        dd|di          S |                     ||                    d	          
          }t          |          S |dk    rGt          |                    dd                    }|                     |          }t          |          S |dk    r8| 
                                }t          j        |t          |          d          S |dk    ru|                    d	          }	|	st          j        ddi          S |                     |	t          |                    d                              }t          | ||          S |dv r|                    d          }
d}|dk    rd}n|dk    rd}
n|dk    rd}
n|
pd}
|                    d          }|                    d          pd }|r|d!         |d!         |d         fnd \  }}|                     ||nd |||
pd||                    d"          #          }t          | ||          S |d$k    r|                     |                    d%          |                    d&          |                    d'          rt!          |d'                   nd |                    d(          rt!          |d(                   nd |                    dd          |                    d"          )          }t          | ||          S |d*k    r|                    d          pd }|                     |                    d+d,          t%          |                    d-d.                    |                    d          |r|d!         |d!         nd |r|d         |d         nd |                    d"          /          }t          | ||          S |d0k    r:|                     |                    d1d2                    }t          | ||          S |d3k    r:|                     |                    d4d2                    }t          | ||          S |d5k    ru|                    d6          }|t          j        dd7i          S |                     t          |          |                    d          8          }t          | ||          S t          j        dd9|i          S ):Ncapture_afterr   rn   rm   >   axrm   visionr   z	bad mode z; use som|vision|axro   rq   r   secondsg      ?r   )appscountr   zfocus_app requires `app`r   )r   >   r   r   r    r!   button   r       r   rightr!   middleleftr   r   )NNr   	modifiers)r   xyr   click_countr   r   r   r   r   r   )r   r   from_xyto_xyr   r   r   r   downr   r   )r   r   r   r   r   r   r   r>   rM   r   r+   r   valuezset_value requires `value`)r   r   zunknown action )rj   r9   r,   r   r   r   _capture_responsefloatr   _text_responser   r   r   _maybe_follow_capturer   r   r   r   intr   r   r   )r   r{   r   r   rn   capr   resr   ro   r   r   r   r   r   r   r   s                    r   r   r   8  s-   /2233M488FE**++...:w(OD(O(O(OPQQQoo4TXXe__o== %%%C0011ll7##c"""  ""z4#d))<<===hhuoo 	E:w(BCDDD$txx7O7O2P2PQQ$Wc=AAAIII(##^##KK}$$FF~%%FF%vF((9%%&&6,',Wq1Ea%(##<1mm&2GG1V-v;hh{++  
 

 %Wc=AAAll.11xx--6:hh?P6Q6Q[E$01222W[26((?2K2KU%_-...QU88Hf--hh{++  
 
 %Wc=AAA&&6,nnhh{F33txx!,,--HHY''BE!H$8eAhhdBE!H$8eAhhdhh{++  
 
 %Wc=AAA 4 455$Wc=AAAkk$((62..//$Wc=AAA!!=:w(DEFFFc%jj$((9:M:MNN$Wc=AAA:w <& < <=>>>r   r   r	   c                    | j         | j        d}| j        r
| j        |d<   | j        r
| j        |d<   t	          j        |          S )Nry   messagemeta)rz   r{   r   r   r   r   )r   payloads     r   r   r     sN    %(VszBBG
{ ) [	
x #(:gr   r   r
   c           
        t          | j                  }d| j         d| j         d| j         | j        r
d| j         ndz   | j        r
d| j        ndz   t          | j                   dg}|r|                    |           d	                    |          }| j
        r{| j        d	k    rp| j
        d d
         }|                    d          rdnd}dd|dddd| d| j
         idg|| j        | j        | j        t          | j                  | j        ddS t          j        | j        | j        | j        | j        | j        d | j        D             |d          S )Nzcapture mode= r   z app=rM   z window=z interactable element(s):
r      z/9j/z
image/jpegz	image/pngTr>   )r   r>   	image_urlurlzdata:z;base64,)r   r   )rn   rr   rs   ru   	png_bytes)_multimodalcontenttext_summaryr   c                ,    g | ]}t          |          S r0   )_element_to_dict)r4   r   s     r   r6   z%_capture_response.<locals>.<listcomp>  s!    ???Q%a((???r   )rn   rr   rs   ro   rv   ru   r   )_format_elementsru   rn   rr   rs   ro   rv   r   extendjoinrt   
startswithpng_bytes_lenr   r   )r   element_indexsummary_linesr   _b64_prefix_mimes         r   r   r     s   $S\22M;;;39;;sz;; #/37R	1.1.>F*c&***B	H s|777	M  ,]+++ii&&G
{ 
sx4'' k"1"o + 6 6v > >OK11$$&Je&J&JS[&J&JKM M
 $ X	SZ!$S\!2!2ART T

 

 
	
 :*w(??#,???    r   
do_capturerj   c                   |st          |          S 	 |                     d          }nA# t          $ r4}t                              d|           t          |          cY d }~S d }~ww xY wt          |          }t          |t                    rw|                    d          rbd|j	         d|j
         |j        r
d|j         ndz   }|d	z   |d
         d         d         z   |d
         d         d<   |d	z   |d         z   |d<   |S 	 t          j        |          }n # t          t          j        f$ r d|i}Y nw xY w|j	        |d<   |j
        |d<   |j        r
|j        |d<   t          j        |          S )Nrm   )rn   zfollow-up capture failed: %sr   [z] ok=u    — rM   z

r   r   r>   r   r   r{   rz   r   )r   r   r[   r   r   r   
isinstancedictr9   r{   rz   r   r   loads	TypeErrorJSONDecodeErrorr   )r   r   r   r   r   respprefixdatas           r   r   r     s     #c"""#oo5o)) # # #5q999c""""""""# S!!D$ $((="9"9 .SZ..cf..3;2^2G#+2G2G2G\^_%+f_tIq7I&7Q%QY6"%n1EE^!z$t+, ! ! !4 !ZDNDJ
{ &+Y:ds,   * 
A()A#A(#A(D D98D9(   ru   List[UIElement]	max_linesr   	List[str]c                l   g }| d |         D ]j}|j                             dd          d d         }|                    d|j         d|j         d|d|j         |j        rd|j         dndz              kt          |           |k    r)|                    d	t          |           |z
   d
           |S )Nr   r   r   z  #z @ z []rM   z  ... +z( more (call capture with app= to narrow))labelreplacerw   indexroleboundsro   r   )ru   r  outr   r  s        r   r   r     s    Cjyj! 7 7c**3B3/

BBB16BBEBBBB'(u4m15mmmm"6 	7 	7 	7 	7
8}}y  

`S]]Y6```aaaJr   r   r   c                ^    | j         | j        | j        t          | j                  | j        dS )N)r  r	  r  r
  ro   )r  r	  r  listr
  ro   )r   s    r   r   r     s0    qx..u  r   c                 F    t           j        dk    rdS ddlm}   |             S )zReturn True iff computer_use can run on this host.

    Conditions: macOS + cua-driver binary installed (or override via env).
    darwinFr   cua_driver_binary_available)sysplatformrS   r  r  s    r   check_computer_use_requirementsr    s7    
 |xuJJJJJJ&&(((r   c                     ddl m}  | S )Nr   COMPUTER_USE_SCHEMA)tools.computer_use.schemar  r  s    r   get_computer_use_schemar    s    ======r   r   )r+   r,   r   r-   )r>   r,   r   r?   )r   r   )r   r   r   r   )r{   r,   r   r   r   r?   )r{   r,   r   r   r   r,   )r   r   r{   r,   r   r   r   r   )r   r	   r   r,   )r   r
   r   r   )r   r   r   r	   r   rj   r   r   )r   )ru   r  r  r   r   r  )r   r   r   r   r   )r   r   )8r   
__future__r   r   loggingrQ   r:   r  	threadingtypingr   r   r   r   r   tools.computer_use.backendr	   r
   r   r   	getLoggerr   r   r   r   r-   _SAFE_ACTIONSr   r   r8   r=   compile
IGNORECASErA   rE   LockrP   rF   __annotations__r\   rG   rH   rX   r]   rT   r   r   r   r   r   r   r   r   r   r  r  r0   r   r   <module>r%     s    @ # " " " " "   				 				 



     3 3 3 3 3 3 3 3 3 3 3 3 3 3            
	8	$	$      	:::;; !y " " "    I+++,,I,,,--I"""##I###$$I---..  !VHU[cdd    BJ'77BJ%r}55BJ'77BJ%r}55BJ$bm44BJ/??     	  )- - - - - SUU           *9 *9 *9 *9 *9% *9 *9 *9b1> 1> 1> 1>hE E E E8   0U? U? U? U?x   & & & &R   :       ) ) ) )     r   