o jVf@sdZddlZddlZddlmZddlmZmZddlm Z ddl m Z ddl m Z mZddlZddlmZd d lmZmZmZmZmZmZmZd d lmZmZd d lmZmZm Z m!Z!m"Z"m#Z#d dl$m%Z%e#ddZ&e ddGdddZ'e(dZ)hdZ*hdZ+hdZ,e(dZ-dddddZ.iZ/e0e,D]Z1e1e/e1<e1de/e1d<e1de/e1d <qe d!e/e2e3d"Z4e ddGd#d$d$Z5e6eedBfZ7e8ee9e2fZ:d%eee9efd&e5fd'd(Z;d%eee9efd&dfd)d*Zd&e6e?e7e:ffd/d0Z@d1e2d2e:d&eeedBeAge>ffd3d4ZBd5eAdBd6e2d7eAd&e>fd8d9ZCd:e2d&e6ee7ge6ed;ffe>ffde?e2d&e'fd?d@ZEe&jFdAgdBdC  D   dqdEe e2dBejGdFdfdGe e>ejGdHdfdIe e?e2dBejGdJdKdLdfdMe e4dBejGdNdfdOe eHdBejGdPdfd&df dQdRZIe&jFgdSdC  D Ddrd>e e?e2ejJdTdfdEe e2dBejGdFdfdUe e>ejGdVdWdXdfdYe e>ejGdZdfd&df d[d\ZKe&jFd]d^gdC  D DdrdEe e2dBejGdFdfdUe e>ejGdVdWdXdfdYe e>ejGdZdfd&dfd_d`ZLe&jFgdadCejMddddDdDdfdbedcedde dEe e2dBejGdedfdfe e2dBejGdgdfdhe e>ejGdidjdfdke e>ejGdldmdfdne!d&dfdodpZNdS)szHContains the 'hf cache' command group with cache management subcommands.N) defaultdict)CallableMapping) dataclass)Enum) AnnotatedAny)CLIError)ANSICachedRepoInfoCachedRevisionInfo CacheNotFound HFCacheInfo _format_sizescan_cache_dir)parse_duration parse_size) RepoIdArg RepoTypeOpt RevisionOptTokenOpt get_hf_api typer_factory)outzManage local cache directory.)helpT)frozenc@s>eZdZUeeed<eeeefed<e edfed<dS)_DeletionResolution revisionsselected.missingN) __name__ __module__ __qualname__ frozensetstr__annotations__dictr r tupler*r*L/home/kuhnn/.local/lib/python3.10/site-packages/huggingface_hub/cli/cache.pyr(s rzA^(?P[a-zA-Z_]+)\s*(?P==|!=|>=|<=|>|<|=)\s*(?P.+)$><=>!=<=>=>refssizetypeaccessedmodified>namer3r5r6z.^(?P[a-zA-Z_]+)(?::(?Pasc|desc))?$descasc)r5r6r3r7z:asc_ascz:desc_desc SortOptions)r4modulec@s*eZdZUdZeed<eed<eed<dS)CacheDeletionCountsz>Simple counters summarizing cache deletions for CLI messaging. repo_countpartial_revision_counttotal_revision_countN)r"r#r$__doc__intr'r*r*r*r+r>Gs  r>selected_by_reporeturncCsdd}d}d}|D]\}}|t|7}t|t|jkr'|d7}|t|7}q ||}t|||S)z(Summarize deletions across repositories.rr)itemslenrr>)rDr?total_revisionsrevisions_in_full_reposreporr@r*r*r+summarize_deletionsTs   rKc Cst|dddD]P}|jd|j}t||ddd}t|t|jkr2td|dq td|d|D]}d t|j pHd }td |j d |d |j q=q dS)zBPretty-print selected cache revisions during confirmation prompts.cS|j|jfSN repo_typerepo_idlowerrJr*r*r+hz0print_cache_selected_revisions..key/cS|jSrM commit_hashrevr*r*r+rSj - z (entire repo): z (detached)z z [z] N) sortedkeysrOrPrGrrtextjoinr2rZsize_on_disk_str)rDrJrepo_keyrrevisionr2r*r*r+print_cache_selected_revisionsfs"rh hf_cache_infocCsLi}i}|jD]}|j}|||<|jD] }||f||j<qq||fS)zPCreate lookup tables so CLI commands can resolve repo ids and revisions quickly.)reposcache_idrQrrZ)ri repo_lookuprevision_lookuprJrfrgr*r*r+build_cache_indexus   rninclude_revisionscCsg}i}t|jddd}|D]+}tdd|jD||<|r3t|jdddD] }|||fq(q||dfq|rI|jddd||fS|jd dd||fS) z;Flatten cache metadata into rows consumed by `hf cache ls`.cSrLrMrNrRr*r*r+rSrTz'collect_cache_entries..rUcSsh|] }|jD]}|qqSr*)r2).0rgrefr*r*r+ sz(collect_cache_entries..cSrXrMrYr[r*r*r+rSr]NcSs&|dj|ddur|djfSdfS)Nrr)rkrZentryr*r*r+rSs cSs |djS)Nr)rkrtr*r*r+rSs )rarjr%rappendsort)riroentries repo_refs_map sorted_reposrJrgr*r*r+collect_cache_entriess"r{exprrycst|}|std|d|d|d|dtvr8tdd|dttd tvrLtd d |dttd d kr[t fd dSdvrzt dt dt dBdt dtffdd }|Sdkrdkrtdddt dt dBdt dtffdd }|Sdkrtdd dt dt dBdt dtffdd }|S) zqConvert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it.zInvalid filter expression: 'z'.rVopvaluezUnsupported operator 'z ' in filter ''. Must be one of .zUnsupported filter key '' in 'r3cs"t|dur |jS|jSrM)_compare_numeric size_on_diskrJrg_)r}size_thresholdr*r+rSs z&compile_cache_filter..>r5r6rJrgNnowrEcs>dkr|jn |dur|jn|j}|durdSt||S)Nr5F) last_accessed last_modifiedr)rJrgr timestamp)rVr}secondsr*r+ _time_filtersz*compile_cache_filter.._time_filterr4r-z/Only '=' is supported for 'type' filters. Got 'rcs|jkSrM)rOrQr)expectedr*r+ _type_filtersz*compile_cache_filter.._type_filterz.Only '=' is supported for 'refs' filters. Got cs2|dur|jn|t}dd|DvS)NcSsg|]}|qSr*)rQ)rprqr*r*r+ z>compile_cache_filter.._refs_filter..)r2getr%rQ)rJrgrr2)ry value_rawr*r+ _refs_filtersz*compile_cache_filter.._refs_filter)_FILTER_PATTERNmatchstrip ValueErrorgrouprQ_ALLOWED_OPERATORSlist _FILTER_KEYSrrr r floatbool)r|ryrrrrr*)rrVr}ryrrrr+compile_cache_filters6    & "$rleftr}rightcCsT|durdS||k||k||k||k||k||kd}||vr&td|||S)z)Evaluate numeric comparisons for filters.NF)r-r/r.r,r1r0z)Unsupported numeric comparison operator: )r)rr}r comparisonsr*r*r+rs r sort_expr.cst|}|std|d|d|d}tvr3tdd|dttd|r7|nt}|d k}d t d t t d fffd d }||fS)zConvert a `hf cache ls` sort expression into a key function for sorting entries. Returns: A tuple of (key_function, reverse_flag) where reverse_flag indicates whether to sort in descending order (True) or ascending order (False). zInvalid sort expression: 'z5'. Expected format: 'key' or 'key:asc' or 'key:desc'.rVorderzUnsupported sort key 'rrrr8rurE.cs|\}}dkr|j}|fSdkr$|dur|j}|fS|j}|fSdkr8|jdur3|j}|fSd}|fSdkr]|durP|jdurK|j}|fSd}|fS|jdurX|jnd}|fStd)Nr7r3r5gr6zUnsupported sort key: )rkrQrrrr)rurJrgr~rUr*r+ _sort_key s. z%compile_cache_sort.._sort_key) _SORT_PATTERNrrrQrr _SORT_KEYSr_SORT_DEFAULT_ORDER CacheEntryr)r)rrexplicit_orderrreverserr*rUr+compile_cache_sorts  rtargetscCst|\}}tt}t}g}|D]V}|}|sq|} td| rE|| } | dur3||q| \} } ||  | | | j q|| } | durT||q| j D]} ||  | | | j qWqdd| D}t t||t|dS)z8Resolve the deletion targets into a deletion resolution.z[0-9a-fA-F]{40}NcSsi|] \}}|t|qSr*)r%)rprJrevsr*r*r+ Lsz-_resolve_deletion_targets..rr r!)rnrsetrrQre fullmatchrrvaddrZrrFrr%r))rirrlrmr rr! raw_targettargetloweredrrJrg matched_repofrozen_selectedr*r*r+_resolve_deletion_targets+s@        rz list | ls)z hf cache lszhf cache ls --revisionsz*hf cache ls --filter "size>1GB" --limit 20zhf cache ls --format json)examplesF cache_dirz9Cache directory to scan (defaults to Hugging Face cache).rzCInclude revisions in the output instead of aggregated repositories.filterz-fz--filterzZFilter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.rwaSort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). Defaults: 'accessed', 'modified', 'size' default to 'desc' (newest/biggest first); 'name' defaults to 'asc' (alphabetical).limitzSLimit the number of results returned. Returns only the top N entries after sorting.c s&zt|}Wnty}z td|j|d}~ww|pg}t||d\}z fdd|D} WntyG}ztt||d}~wwt | D] fdd|D}qN|rzt |j \} } |j | | dWnty}ztt||d}~ww|dur|dkrtd |d |d|}|rd d|D} t j| gd d ddidnfdd|D} t j| gddddid|rdd|D} t| }|rtdd|D}tdd|D}ntdd| D}tdd| D}t td|d|dt|ddSdS)z&List cached repositories or revisions.Cache directory not found: N)rocsg|]}t|qSr*)r)rpr|ryr*r+rszls..cs$g|]}|d|dr|qS)rrr*rpru)fnrr*r+rs$)rVrrz&Limit must be a positive integer, got rc SsFg|]\}}|dur|j|j|j|jt|j|j|jt|j dqS)N)idrPrOrg snapshot_pathr3rr2) rkrPrOrZr&rrelast_modified_strrar2)rprJrgr*r*r+rs  )rrgr3rr2rgr3r)headersid_key alignmentsc sBg|]\}}|j|j|j|j|jpd|jt|tdqS)rs)rrPrOr3rrr2) rkrPrOrelast_accessed_strrrarr%rprJrrr*r+rs )rr3rrr2rcSsh|]\}}|qSr*r*rr*r*r+rrrzls..css |] \}}|durdVqdS)Nrr*rprr\r*r*r+ szls..css"|] \}}|dur|jVqdSrMrrr*r*r+r css|]}t|jVqdSrM)rGrrprJr*r*r+rcs|]}|jVqdSrMrrr*r*r+rz Found z repo(s) for a total of z revision(s) and z on disk.)rrr rr{rtyper BadParameterr&timerr~rwrtablerGsumrcr boldr)rrrrwrriexcfiltersrx filter_fns sort_key_fnrrF unique_reposr?revision_count total_sizer*)rrryr+lsWs/      r)zhf cache rm model/gpt2zhf cache rm z hf cache rm model/gpt2 --dry-runzhf cache rm model/gpt2 --yeszQOne or more repo IDs (e.g. model/bert-base-uncased) or revision hashes to delete.yesz-yz--yeszSkip confirmation prompt.dry_runz,Preview deletions without removing anything.c Cszt|}Wnty}z td|j|d}~wwt||}|jr7ddd|jD}td|t |j dkrIt dt j dd |jt|j }t|j} g} | jrd| | jd | jrp| | jd | s{| | jd d | } t d | d|jdt|j|rtjdd| j| j|jddStjd|d|t|j} tjd| jd| jd|jd| j| j|jddS)z(Remove cached repositories or revisions.rN cs|]}d|VqdSr^Nr*rr*r*r+r rzrm..zCould not find in cache: rzNothing to delete.codez repo(s)z revision(s)z and About to delete z totalling rDry run: no files were deleted.T)rrjrr3zProceed with deletion?rDeleted z repo(s) and z revision(s); freed ) repos_deletedrevisions_deletedfreed)rrr rrr!rdrwarningrGrrcrExitdelete_revisionsrarKr r?rvr@rAexpected_freed_size_strrhresultconfirmexecute) rrrrrir resolutiondetailsstrategycounts summary_parts summary_textr*r*r+rmsZ%         rzhf cache prunezhf cache prune --dry-runc CsJzt|}Wnty}z td|j|d}~wwi}t}|jD]}tdd|jD}|s3q$|||<|dd|Dq$t |dkrOt ddSt t||dd } |j t| j} t|} t d | jd | jd t||rt jd d| j| jddSt jd|d| t jd| jd| jd| j| jddS)z)Remove detached revisions from the cache.rNcss"|] }t|jdkr|VqdS)rN)rGr2rprgr*r*r+rYrzprune..csrrMrYrr*r*r+r]rrz2No unreferenced revisions found. Nothing to prune.r*rrz unreferenced revision(s) (z total).rT)rrr3zProceed?rrz! unreferenced revision(s); freed r)rr)rrr rrrjr%rupdaterGrrcrrrarKrArrhrrr) rrrrirr rrJdetachedrrrr*r*r+prune9sV     r)zhf cache verify gpt2z)hf cache verify gpt2 --revision refs/pr/1z.hf cache verify my-dataset --repo-type datasetrPrOrgzXCache directory to use when verifying files from cache (defaults to Hugging Face cache). local_dirz?If set, verify files under this directory instead of the cache.fail_on_missing_filesz--fail-on-missing-filesz?Fail if some files exist on the remote but are missing locally.fail_on_extra_filesz--fail-on-extra-fileszLFail if some files exist locally but are not present on the remote revision.tokenc Cs|dur|durtdtjddt|d}|j|t|dr#|jnt|||||d} d} | j rGd d d | j D} t d | d } | j rm|rbd dd | j D} t d| d } n t t| j d| jr|rd dd | jD} t d| d } n t t| jd| j} | dkrtd|d|jd| d| jtj| dtjd| jd|jd|d||j| jt| ddS)aVerify checksums for a single repo revision from cache or a local directory. Examples: - Verify main revision in cache: `hf cache verify gpt2` - Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1` - Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset` - Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo` NzCCannot pass both --local-dir and --cache-dir. Use one or the other.r r)rr~)rPrOrgrrrrrc ss<|]}d|dd|dd|dd|dVqdS) r^pathz : expected rz ( algorithmz), got actualNr*)rpmr*r*r+rs * zverify..u<❌ Checksum verification failed for the following file(s): rcsrrr*rppr*r*r+rru6❌ Missing files (present remotely, absent locally): zM remote file(s) are missing locally. Use --fail-on-missing-files for details.csrrr*rr*r*r+rru4❌ Extra files (present locally, absent remotely): zV local file(s) do not exist on the remote repo. Use --fail-on-extra-files for details.zVerification failed for 'z' (z) in z. Revision: z Verified z file(s) for z 'z'. All checksums match.)rPrOcheckedr)rerrorrrrverify_repo_checksumshasattrr~r& mismatchesrdrc missing_pathsrrG extra_paths verified_pathrgr checked_count) rPrOrgrrrrrapir exit_coderverified_locationr*r*r+verifys`0       r)NFNNN)NFF)OrBrr collectionsrcollections.abcrr dataclassesrenumrtypingrrrhuggingface_hub.errorsr utilsr r r rrrrutils._parsingrr _cli_utilsrrrrrr_outputr cache_clircompilerrrrrr_sort_options_dictrarVr&r"r<r>r)rr(r% RepoRefsMaprKrhrnrrr{rrrrrcommandOptionrCrArgumentrrmodelrr*r*r*r+s     $           A,6,  $   R I