o j+7@sdZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z mZddlmZdd lmZd d lmZmZmZmZmZmZd d lmZd d lmZmZe r^d dlm Z e!e"Z#e$dZ%dZ&dddZ'eGdddZ(e    dCdee(de)dBde)de)de)dBde)dBde*e)e)fdBde+e)dBde,e+e*e+e*e)dBffddZ-Gd d!d!eZ.Gd"d#d#eZ/   dDd$d%d&e*de)dBde*e)e)fdBde)dBddf d'd(Z0d)e*fd*d+Z1d,e*fd-d.Z2d/e*fd0d1Z3d$d%d2e)ddfd3d4Z4d$d%d5e*d6e5d2e)ddf d7d8Z6d5e*d9e(d6e5de+e)fd:d;Z7dd?Z8d$d%d@e+e)d6e5de+e*fdAdBZ9dS)Ez.Git LFS related type definitions and utilitiesN)Iterable) dataclass)ceil)getsize) TYPE_CHECKINGBinaryIO TypedDict)unquote) constants)build_hf_headersfix_hf_endpoint_in_urlhf_raise_for_status http_backoffloggingvalidate_hf_hub_args) SliceFileObj)sha256 sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)Acceptz Content-Typec@s`eZdZUdZeed<eed<eed<edefddZ edefd d Z ed e fd d Z dS) UploadInfoad Dataclass holding required information to determine whether a blob should be uploaded to the hub using the LFS protocol or the regular protocol Args: sha256 (`bytes`): SHA256 hash of the blob size (`int`): Size in bytes of the blob sample (`bytes`): First 512 bytes of the blob rsizesamplepathcCsZt|}t|d}|ddd}t|}Wdn1s!wY||||dS)Nrbrrr)ropenpeekr)clsrrfilershar#F/home/kuhnn/.local/lib/python3.10/site-packages/huggingface_hub/lfs.py from_pathIs   zUploadInfo.from_pathdatacCs&t|}|t||dd|dS)Nr)rrr)rdigestlen)r r&r"r#r#r$ from_bytesQs zUploadInfo.from_bytesfileobjcCsD|d}|dtjt|}|}|dtj||||dS)Nrrr)readseekioSEEK_SETrtell)r r*rr"rr#r#r$ from_fileobjVs zUploadInfo.from_fileobjN) __name__ __module__ __qualname____doc__bytes__annotations__int classmethodstrr%r)rr0r#r#r#r$r6s  r upload_infostoken repo_typerepo_idrevisionendpointheaders transfersreturncCs|dur|ntj}d}|tjvrtj|}|d||d} d|dur&|nddgdd |Dd d } |dur?d t|i| d <itt|d|pJi}td| || d} t| | } | dd} t | t skt d| d}t |t rw|nd}dd | Ddd | D|fS)a Requests the LFS batch endpoint to retrieve upload instructions Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md Args: upload_infos (`Iterable` of `UploadInfo`): `UploadInfo` for the files that are being uploaded, typically obtained from `CommitOperationAdd.upload_info` repo_type (`str`): Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. repo_id (`str`): A namespace (user or an organization) and a repo name separated by a `/`. revision (`str`, *optional*): The git revision to upload to. headers (`dict`, *optional*): Additional headers to include in the request transfers (`list`, *optional*): List of transfer methods to use. Defaults to ["basic", "multipart"]. Returns: `LfsBatchInfo`: 3-tuple: - First element is the list of upload instructions from the server - Second element is a list of errors, if any - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet") Raises: [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) If an argument is invalid or the server response is malformed. [`HfHubHTTPError`] If the server returned an error. N/z.git/info/lfs/objects/batchuploadbasic multipartcSsg|] }|j|jdqS)oidr)rhexr).0rEr#r#r$ s z'post_lfs_batch_info..r) operationrAobjects hash_algonameref)r;POSTr@jsonrNzMalformed response from servertransfercSsg|] }d|vrt|qSerror)_validate_batch_actionsrKobjr#r#r$rLcSsg|] }d|vrt|qSrV)_validate_batch_errorrYr#r#r$rLr[)r ENDPOINTREPO_TYPES_URL_PREFIXESr LFS_HEADERSr rrrTget isinstancelist ValueErrorr9)r:r;r<r=r>r?r@rA url_prefix batch_urlpayloadresp batch_inforNchosen_transferr#r#r$post_lfs_batch_info`sB,        rjc@seZdZUeed<eed<dS) PayloadPartT partNumberetagN)r1r2r3r7r6r9r#r#r#r$rks  rkc@s&eZdZUdZeed<eeed<dS)CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rIpartsN)r1r2r3r4r9r6rbrkr#r#r#r$rns rnrMrlfs_batch_actionc CsFt||d}|durtd|jddS|dd}t||dd}|dur2t||di}|d} t|d |d } | durjzt| } Wntt fy`td | d wt ||| | d nt || d|durt|t|d |} t d| t ||d|jj|jjdd} t| t|jddS)a9 Handles uploading a given object to the Hub with the LFS protocol. Can be a No-op if the content of the file is already present on the hub large file storage. Args: operation (`CommitOperationAdd`): The add operation triggering this upload. lfs_batch_action (`dict`): Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for more details. headers (`dict`, *optional*): Headers to include in the request, including authentication and user agent headers. Raises: [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) If `lfs_batch_action` is improperly formatted [`HfHubHTTPError`] If the upload resulted in an error actionsNzContent of file z. is already present upstream - skipping uploadrEverifyheader chunk_sizehref)r?zTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.)rMrsrt upload_url)rMrvrR)r;r@rHrSz: Upload successful)rXr`loggerdebug path_in_repo_validate_lfs_actionr r7rc TypeError_upload_multi_part_upload_single_partrr upload_inforrJrr) rMrpr;r@r?rq upload_action verify_actionrsrtrv verify_url verify_respr#r#r$ lfs_uploadsD        r lfs_actioncCs:t|dtr|ddust|dtstd|S).validates response from the LFS batch endpointrursNz"lfs_action is improperly formatted)rar`r9dictrc)rr#r#r$rz s rzlfs_batch_actionscCspt|dtrt|dtstd|did}|did}|dur.t||dur6t||S)rrIrz)lfs_batch_actions is improperly formattedrqrErrN)rar`r9r7rcrz)rrrr#r#r$rXs rXlfs_batch_errorcCsht|dtrt|dtstd|d}t|tr.t|dtr.t|dts2td|S)rrIrz'lfs_batch_error is improperly formattedrWmessagecode)rar`r9r7rcr)r error_infor#r#r$r\"s  r\rvcCsH|jdd}td||d}t|WddS1swYdS)aK Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol) Args: upload_url (`str`): The URL to PUT the file to. fileobj: The file-like object holding the data to upload. Raises: [`HfHubHTTPError`] If the upload resulted in an error. T with_tqdmPUTr&N)as_filerr)rMrvr*responser#r#r$r}0s "r}rsrtcCsHt||j|d}t|||d}td|t||jjtd}t|dS)z@ Uploads file using HF multipart LFS transfer protocol. )rsr~rt)rMsorted_parts_urlsrtrR)rTr@N) _get_sorted_parts_urlsr~_upload_parts_iterativelyr_get_completion_payloadrrJr_r)rMrsrtrvrresponse_headerscompletion_resr#r#r$r|Ds r|r~cCsNddtdd|DdddD}t|}|t|j|kr%td|S)NcSsg|]\}}|qSr#r#)rK_rvr#r#r$rL\sz*_get_sorted_parts_urls..cSs2g|]\}}|rt|dkrt|d|fqS)r )isdigitr(r7)rKpart_numrvr#r#r$rL_s  cSs|dS)Nrr#)tr#r#r$dsz(_get_sorted_parts_urls..)keyz0Invalid server response to upload large LFS file)sorteditemsr(rrrc)rsr~rtsorted_part_upload_urls num_partsr#r#r$r[s rrrIcCsfg}t|D]'\}}|d}|dus|dkr#td|d|d||d|dq||dS)NrmrCzInvalid etag (`z`) returned for part r )rlrm)rIro) enumerater`rcappend)rrIro part_numberrsrmr#r#r$rms  rrc Csg}|jdd<}t|D].\}}t||||d}td||d}t|||jWdn1s6wYq Wd|S1sGwY|S)NTr) seek_from read_limitrr)rrrrrrr@) rMrrtr@r*part_idxpart_upload_url fileobj_slicepart_upload_resr#r#r$r|s&   r)NNNN)NNN):r4r-recollections.abcr dataclassesrmathros.pathrtypingrrr urllib.parser huggingface_hubr utilsr r rrrr utils._lfsr utils.sharr _commit_apir get_loggerr1rwcompile OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDr_rr9rrbtuplerjrkrnrrzrXr\r}r7r|rrrr#r#r#r$s            )  V  G