
    "hjG                     >   d Z ddlZddlmZ ddlmZmZ ddlmZ ddlmZ ddlmZ dd	lm	Z	 dd
lm
Z
 ddlmZ ddlmZ ddlmZ ddl
mZ ddlmZmZmZmZ ddlmZ ddl ddlZddlZddlZddlZddlZ ej:                  e      Z	 	 	 	 	 	 	 d0dZ 	 	 	 	 	 	 	 d0dZ!dZ"	 d1dZ#d Z$d Z%d2dZ&d2dZ'd2dZ( G d de)      Z* G d de)      Z+ G d d e*      Z, G d! d"e*      Z-d#Z.d$Z/ G d% d&e)      Z0d' Z1 G d( d)e0      Z2 G d* d+e0      Z3d2d,Z4d2d-Z5 G d. d/e)      Z6y)3uW   
oss2.resumable
~~~~~~~~~~~~~~

该模块包含了断点续传相关的函数和类。
    N   )utils)b64encode_as_stringb64decode_from_string)	iterators)
exceptions)defaults)http)models)CryptoBucket)Bucket)PartIterator)PartInfo)json	stringify
to_unicode	to_string)	TaskQueue)*c
                 (   t         j                  dj                  | j                  t	        |      |||||             t
        j                  j                  |      }
t        j                  |t        j                        }t         j                  dj                  |
|             |
|k\  r't        | |||
||||||	
      }|j                         }|S t        t        |      d      5 }| j                  ||||      }ddd       |S # 1 sw Y   S xY w)u  断点上传本地文件。

    实现中采用分片上传方式上传本地文件，缺省的并发数是 `oss2.defaults.multipart_num_threads` ，并且在
    本地磁盘保存已经上传的分片信息。如果因为某种原因上传被中断，下次上传同样的文件，即源文件和目标文件路径都
    一样，就只会上传缺失的分片。

    缺省条件下，该函数会在用户 `HOME` 目录下保存断点续传的信息。当待上传的本地文件没有发生变化，
    且目标文件名没有变化时，会根据本地保存的信息，从断点开始上传。

    使用该函数应注意如下细节：
        #. 如果使用CryptoBucket，函数会退化为普通上传

    :param bucket: :class:`Bucket <oss2.Bucket>` 或者 ：:class:`CryptoBucket <oss2.CryptoBucket>` 对象
    :param key: 上传到用户空间的文件名
    :param filename: 待上传本地文件名
    :param store: 用来保存断点信息的持久存储，参见 :class:`ResumableStore` 的接口。如不指定，则使用 `ResumableStore` 。

    :param headers: HTTP头部
        # 调用外部函数put_object 或 init_multipart_upload传递完整headers
        # 调用外部函数uplpad_part目前只传递OSS_REQUEST_PAYER, OSS_TRAFFIC_LIMIT
        # 调用外部函数complete_multipart_upload目前只传递OSS_REQUEST_PAYER, OSS_OBJECT_ACL
    :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict

    :param multipart_threshold: 文件长度大于该值时，则用分片上传。
    :param part_size: 指定分片上传的每个分片的大小。如不指定，则自动计算。
    :param progress_callback: 上传进度回调函数。参见 :ref:`progress_callback` 。
    :param num_threads: 并发上传的线程数，如不指定则使用 `oss2.defaults.multipart_num_threads` 。

    :param params: HTTP请求参数
        # 只有'sequential'这个参数才会被传递到外部函数init_multipart_upload中。
        # 其他参数视为无效参数不会往外部函数传递。
    :type params: dict
    zStart to resumable upload, bucket: {0}, key: {1}, filename: {2}, headers: {3}, multipart_threshold: {4}, part_size: {5}, num_threads: {6}z<The size of file to upload is: {0}, multipart_threshold: {1})	part_sizeheadersprogress_callbacknum_threadsparamsrb)r   r   N)loggerdebugformatbucket_namer   ospathgetsizer	   getmultipart_threshold_ResumableUploaderuploadopenr   
put_object)bucketkeyfilenamestorer   r%   r   r   r   r   sizeuploaderresultfs                 L/root/.hermes/SkillClaw/.venv/lib/python3.12/site-packages/oss2/resumable.pyresumable_uploadr3   &   s   R LL MMSVTZTfTfhqruhvT\^egzT]_jNlm 77??8$D",,':H<X<XY
LLOVVW[]pqr""%fc8T509.58I2=-35 "
 M *X&- 	e&&sAwRc&dF	e M	e Ms   'DDc
                 H   t         j                  dj                  | j                  t	        |      ||||             t        j                  |t
        j                        }t        |	t        t        g      }
| j                  |||
      }t         j                  dj                  |j                  |             |j                  |k\  rDt        | ||t        j                  |      ||||||

      }|j!                  |j"                         y| j%                  |||||
       y)uZ
  断点下载。

    实现的方法是：
        #. 在本地创建一个临时文件，文件名由原始文件名加上一个随机的后缀组成；
        #. 通过指定请求的 `Range` 头按照范围并发读取OSS文件，并写入到临时文件里对应的位置；
        #. 全部完成之后，把临时文件重命名为目标文件 （即 `filename` ）

    在上述过程中，断点信息，即已经完成的范围，会保存在磁盘上。因为某种原因下载中断，后续如果下载
    同样的文件，也就是源文件和目标文件一样，就会先读取断点信息，然后只下载缺失的部分。

    缺省设置下，断点信息保存在 `HOME` 目录的一个子目录下。可以通过 `store` 参数更改保存位置。

    使用该函数应注意如下细节：
        #. 对同样的源文件、目标文件，避免多个程序（线程）同时调用该函数。因为断点信息会在磁盘上互相覆盖，或临时文件名会冲突。
        #. 避免使用太小的范围（分片），即 `part_size` 不宜过小，建议大于或等于 `oss2.defaults.multiget_part_size` 。
        #. 如果目标文件已经存在，那么该函数会覆盖此文件。
        #. 如果使用CryptoBucket，函数会退化为普通下载


    :param bucket: :class:`Bucket <oss2.Bucket>` 或者 ：:class:`CryptoBucket <oss2.CryptoBucket>` 对象
    :param str key: 待下载的远程文件名。
    :param str filename: 本地的目标文件名。
    :param int multiget_threshold: 文件长度大于该值时，则使用断点下载。
    :param int part_size: 指定期望的分片大小，即每个请求获得的字节数，实际的分片大小可能有所不同。
    :param progress_callback: 下载进度回调函数。参见 :ref:`progress_callback` 。
    :param num_threads: 并发下载的线程数，如不指定则使用 `oss2.defaults.multiget_num_threads` 。

    :param store: 用来保存断点信息的持久存储，可以指定断点信息所在的目录。
    :type store: `ResumableDownloadStore`

    :param dict params: 指定下载参数，可以传入versionId下载指定版本文件

    :param headers: HTTP头部,
        # 调用外部函数head_object目前只传递OSS_REQUEST_PAYER
        # 调用外部函数get_object_to_file, get_object目前需要向下传递的值有OSS_REQUEST_PAYER, OSS_TRAFFIC_LIMIT
    :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict

    :raises: 如果OSS文件不存在，则抛出 :class:`NotFound <oss2.exceptions.NotFound>` ；也有可能抛出其他因下载文件而产生的异常。
    z|Start to resumable download, bucket: {0}, key: {1}, filename: {2}, multiget_threshold: {3}, part_size: {4}, num_threads: {5})r   r   z?The size of object to download is: {0}, multiget_threshold: {1})r   r   r   r-   r   r   )r   r   r   N)r   r   r   r    r   r	   r$   multiget_threshold_populate_valid_headersOSS_REQUEST_PAYEROSS_TRAFFIC_LIMIThead_objectcontent_length_ResumableDownloader_ObjectInfomakedownload
server_crcget_object_to_file)r*   r+   r,   r5   r   r   r   r-   r   r   valid_headersr0   
downloaders                r2   resumable_downloadrC   f   s   ^ LL 3396&:L:LiX[n^f:LiYd4fg "&8(:U:UV+G6GIZ5[\MFMJF
LLRYYZ`ZoZoZln o 22)&#xAQAQRXAYen<M[fns17P
 	F--.!!#xCT]c*7 	" 	9    i c                 \    |st         j                  }t        | |t         j                        S )u   确定分片上传是分片的大小。

    :param int total_size: 总共需要上传的长度
    :param int preferred_size: 用户期望的分片大小。如果不指定则采用defaults.part_size

    :return: 分片大小
    )r	   r   _determine_part_size_internalmax_part_count)
total_sizepreferred_sizes     r2   determine_part_sizerJ      s'     !++(^XE\E\]]rD   c                     | |k  r| S ||z  | k  s|t         j                  k  r"|dz  }||z  | k  r|t         j                  k  r"|S )N   )r	   min_part_size)rH   rI   	max_counts      r2   rF   rF      sX    N"
9
$z
1^hF\F\5\'!+ 9
$z
1^hF\F\5\ rD   c                     g }t        j                  | |      }t        |      D ];  }||dz
  k(  r||z  }| }n
||z  }||z   }|j                  t	        |dz   ||             = |S )Nr   )r   how_manyrangeappend_PartToProcess)rH   r   parts	num_partsistartends          r2   _split_to_partsrY      sx    Ez95I9 8	A	MEC	MEe#C^AE5#678 LrD   c                     | |yt        j                  |       } t        j                         }|D ]  }| j                  |      | |   ||<    t        |      dk(  rd}|S )u8  构建只包含有效keys的http header

    :param headers: 需要过滤的header
    :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict

    :param valid_keys: 有效的关键key列表
    :type valid_keys: list

    :return: 只包含有效keys的http header, type: oss2.CaseInsensitiveDict
    Nr   )r
   CaseInsensitiveDictr$   len)r   
valid_keysrA   r+   s       r2   r6   r6      su     *,&&w/G,,.M .;;s'!(M#. =QrD   c                     | |yt        j                  |       } | j                         }|D ]%  }|j                  |      |j	                  |       ' t        |      dk(  rd}|S )u9  过滤无效keys的http header

    :param headers: 需要过滤的header
    :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict

    :param invalid_keys: 无效的关键key列表
    :type invalid_keys: list

    :return: 过滤无效header之后的http headers, type: oss2.CaseInsensitiveDict
    Nr   )r
   r[   copyr$   popr\   )r   invalid_keysrA   r+   s       r2   _filter_invalid_headersrb      st     ,.&&w/GLLNM #S!-c"# =QrD   c                     | |yt               }|D ]  }| j                  |      | |   ||<    t        |      dk(  rd}|S )u   构建只包含有效keys的params

    :param params: 需要过滤的params
    :type params: dict

    :param valid_keys: 有效的关键key列表
    :type valid_keys: list

    :return: 只包含有效keys的params
    Nr   )dictr$   r\   )r   r]   valid_paramsr+   s       r2   _populate_valid_paramsrf   	  s]     ~+6L ,::c?& &sL, <ArD   c                   .    e Zd Z	 ddZd Zd Zd Zd Zy)_ResumableOperationNc                 H   || _         t        |      | _        || _        || _        t
        j                  j                  |      | _        || _	        |A| j                  j                  |j                  | j                  | j                        | _        nA| j                  j                  |j                  | j                  | j                  |      | _        t        j                  dj                  | j                               t!        j"                         | _        || _        y )Nz)Init _ResumableOperation, record_key: {0})r*   r   r+   r,   r.   r!   r"   abspath_abspath_ResumableOperation__storemake_store_keyr    _ResumableOperation__record_keyr   r   r   	threadingLock_ResumableOperation__plock&_ResumableOperation__progress_callback)selfr*   r+   r,   r.   r-   r   	versionids           r2   __init__z_ResumableOperation.__init__#  s    S> 	1 $ ; ;F<N<NPTPXPXZ^ZgZg hD $ ; ;F<N<NPTPXPXZ^ZgZgir sD@GGHYHYZ[ !~~'#4 rD   c                 N    | j                   j                  | j                         y N)rl   deletern   rs   s    r2   _del_recordz_ResumableOperation._del_record9  s    D--.rD   c                 P    | j                   j                  | j                  |       y rw   )rl   putrn   rs   records     r2   _put_recordz_ResumableOperation._put_record<  s    **F3rD   c                 L    | j                   j                  | j                        S rw   )rl   r$   rn   ry   s    r2   _get_recordz_ResumableOperation._get_record?  s    || 1 122rD   c                     | j                   r2| j                  5  | j                  || j                         d d d        y y # 1 sw Y   y xY wrw   )rr   rq   r.   )rs   consumed_sizes     r2   _report_progressz$_ResumableOperation._report_progressB  sJ    ## C((		BC C $C Cs   A  A	NN)__name__
__module____qualname__ru   rz   r   r   r    rD   r2   rh   rh   "  s    375,/43CrD   rh   c                   "    e Zd Zd Zed        Zy)r<   c                 .    d | _         d | _        d | _        y rw   )r.   etagmtimery   s    r2   ru   z_ObjectInfo.__init__I  s    		
rD   c                     t               }| j                  |_        | j                  |_        | j                  |_        |S rw   )r<   r:   r.   r   last_modifiedr   )head_object_result
objectInfos     r2   r=   z_ObjectInfo.makeN  s7     ]
,;;
,11
-;;
rD   N)r   r   r   ru   staticmethodr=   r   rD   r2   r<   r<   H  s    
  rD   r<   c                   l     e Zd Z	 	 	 	 	 	 d fd	ZddZddZd Zd Zd Zd Z	d Z
d	 Zd
 Zd Z xZS )r;   c           	         d }|	"|	j                  d      |	j                  d      }t        t        |   ||||j                  |xs
 t               ||       || _        d| _        t        j                   |t        j                        | _
        t        | j                  | j                  t              | _
        d | _        t        j                   |t        j                        | _        d | _        d | _        |	| _        |
| _        t)        j*                         | _        d | _        t0        j3                  dj5                  |j6                  t9        |      | j                  | j                               y )N	versionId)r   rt   ResumableDownloadzQInit _ResumableDownloader, bucket: {0}, key: {1}, part_size: {2}, num_thread: {3})r$   superr;   ru   r.   ResumableDownloadStorer   _ResumableDownloader__opr	   multiget_part_size_ResumableDownloader__part_sizerF   _MAX_MULTIGET_PART_COUNT_ResumableDownloader__tmp_filemultiget_num_threads!_ResumableDownloader__num_threads$_ResumableDownloader__finished_parts#_ResumableDownloader__finished_size_ResumableDownloader__params_ResumableDownloader__headersro   rp   _ResumableDownloader__lock_ResumableDownloader__recordr   r   r   r    r   )rs   r*   r+   r,   r   r   r-   r   r   r   r   rt   	__class__s               r2   ru   z_ResumableDownloader.__init__Y  s5    	&**["9"E

;/I"D263*//383T<R<TEV=F 	3 	H %'	#<<	83N3NO8DDTDTVno%\\+x7T7TU $#   nn&hoo	#0@0@$BTBTV 	WrD   c                    | j                          | j                         }t        j                  dj	                  |             t        | j                  d      j                          t        t        j                  | j                  |      | j                  g| j                  z        }|j                          | j                  j                   rEt#        | j$                  d       }t'        j(                  |      }t'        j*                  d||d        t'        j,                  | j                  | j.                         | j1                  | j2                         | j5                          y )NzParts need to download: {0}a)parts_to_downloadc                     | j                   S rw   part_numberps    r2   <lambda>z/_ResumableDownloader.download.<locals>.<lambda>  
     rD   r+   zresume download)!_ResumableDownloader__load_record+_ResumableDownloader__get_parts_to_downloadr   r   r   r(   r   closer   	functoolspartial_ResumableDownloader__producer_ResumableDownloader__consumerr   runr*   
enable_crcsortedr   r   calc_obj_crc_from_parts	check_crcforce_renamer,   r   r.   rz   )rs   r?   r   qrT   
object_crcs         r2   r>   z_ResumableDownloader.downloady  s    88:299:KLM 	T__c"((*i''K\]'$*<*<<>	;;!!4006MNE66u=JOO-z:tL4??DMM:dii(rD   c                 4    |D ]  }|j                  |        y rw   r|   )rs   r   r   parts       r2   
__producerz_ResumableDownloader.__producer  s    % 	DEE$K	rD   c                     |j                         r6|j                         }|y | j                  |       |j                         r5y y rw   )okr$   #_ResumableDownloader__download_partrs   r   r   s      r2   
__consumerz_ResumableDownloader.__consumer  s7    ddf557D|  & ddfrD   c                    | j                  | j                         t        | j                  d      5 }|j	                  |j
                  t        j                         t        | j                  t        t        g      }|t        j                         }| j                  j                  |t         <   t#        j$                  | j                  j&                        |t(        <   | j*                  j-                  | j.                  |j
                  |j0                  dz
  f|| j2                        }t#        j4                  |||j0                  |j
                  z
  |j6                         d d d        j8                  |_        t<        j?                  djA                  |jB                  |j
                  |j0                               | jE                  |       y # 1 sw Y   pxY w)Nzrb+r   )
byte_ranger   r   )
request_idzRdown part success, add part info to record, part_number: {0}, start: {1}, end: {2})#r   r   r(   r   seekrW   r!   SEEK_SETr6   r   r7   r8   r
   r[   r   r   IF_MATCHr   	http_dater   IF_UNMODIFIED_SINCEr*   
get_objectr+   rX   r   copyfileobj_and_verifyr   
client_crcpart_crcr   r   r   r   !_ResumableDownloader__finish_partrs   r   r1   r   r0   s        r2   __download_partz$_ResumableDownloader.__download_part  sm   d223$//5) 
	iQFF4::r{{+-dnn?PRc>deG224 $ 4 4GH+0??4??;P;P+QG'([[++DHH$**dhhYZlA[eluy  vC  vC+  DF((DHHtzz4IV\VgVgh
	i ))ippdjj$((4 	5 	4 !
	i 
	is   D4GGc                 l   | j                         }t        j                  dj                  |             |r8| j	                  |      s't        j                  d       | j                          d }|rut        j                  j                  | j                  |d   z         sFt        j                  dj                  | j                  |d   z                | j                          d }|rv| j                  |      ret        j                  dj                  | j                               t        j                  | j                  |d   z          | j                          d }|s| j                  | j                   j"                  | j                  | j$                  j&                  | j$                  j(                  | j$                  j*                  | j,                  | j.                  | j1                         g d
}t        j                  dj                  | j                   j"                  | j                  | j,                               | j3                  |       | j                  |d   z   | _        |d   | _        t7        d	 |d
   D              | _        t;        d | j8                  D              | _        || _        y )NLoad record return {0}3The content of record is invalid, delete the record
tmp_suffixz0Temp file: {0} does not exist, delete the recorduA   Object: {0} has been overwritten，delete the record and tmp file)
op_typer*   r+   r.   r   r   r   	file_pathr   rT   z5Add new record, bucket: {0}, key: {1}, part_size: {2}r   c              3   P   K   | ]  }t        |d    |d   |d   |d            yw)r   rW   rX   r   N)rS   .0r   s     r2   	<genexpr>z5_ResumableDownloader.__load_record.<locals>.<genexpr>  s3      %lVWN1]+QwZ51Z=Q%ls   $&rT   c              3   4   K   | ]  }|j                     y wrw   r.   r   s     r2   r   z5_ResumableDownloader.__load_record.<locals>.<genexpr>       "Ia166"I   ) r   r   r   r   $_ResumableDownloader__is_record_sanewarnrz   r!   r"   existsr,   '_ResumableDownloader__is_remote_changedr+   r   silently_remover   r*   r    r   r.   r   r   r   rk   $_ResumableDownloader__gen_tmp_suffixr   r   listr   sumr   r   r}   s     r2   __load_recordz"_ResumableDownloader.__load_record  s&   !!#-44V<=$//7KKMNF"''..9M)MNKKJQQ| 446 7Fd..v6KK[bbcgckcklm!!$--&2F"FGF!%dkk6M6MVZV^V^"oo22T__=R=R\`\k\k\p\p#'#3#3$--_c_t_t_v!#F LLPWW''43C3CE FV$--&*>>!+. $ %l[abi[j%l !l""I43H3H"IIrD   c                     | j                   sJ t        t        | j                  | j                              }t        | j
                        }t        t        ||z
        d       S )Nc                     | j                   S rw   r   r   s    r2   r   z>_ResumableDownloader.__get_parts_to_download.<locals>.<lambda>  s
    !-- rD   r   )r   setrY   r.   r   r   r   r   )rs   all_setfinished_sets      r2   __get_parts_to_downloadz,_ResumableDownloader.__get_parts_to_download  sQ    }}}odii1A1ABC4001d7\128OPPrD   c                    	 |d   | j                   k7  r(t        j                  dj                  |d                ydD ]?  }t	        ||   t
              rt        j                  dj                  |||                 y dD ]?  }t	        ||   t              rt        j                  dj                  |||                 y t	        |d   t              s)t        j                  d	j                  ||                y	 y# t        $ r8}t        j                  d
j                  |j                               Y d }~yd }~ww xY w)Nr   1op_type invalid, op_type in record:{0} is invalidF)r   r   r   r*   r+   z{0} is not a string: {1})r   r.   r   z{0} is not an integer: {1}rT   z{0} is not a list: {1}Key not found: {0}T)
r   r   errorr   
isinstancestrintr   KeyErrorargsrs   r~   r+   es       r2   __is_record_sanez%_ResumableDownloader.__is_record_sane  s%   	i DII-PWWX^_hXijkK !!&+s3LL!;!B!B3s!TU !
 6 !!&+s3LL!=!D!DS&QT+!VW !
 fWot45<<S&+NO 5 	  	LL-44QVV<=	s4   9D D )D ?D )D <D 	E.D>>Ec                     |d   | j                   j                  k7  xs: |d   | j                   j                  k7  xs |d   | j                   j                  k7  S )Nr   r.   r   )r   r   r.   r   r}   s     r2   __is_remote_changedz(_ResumableDownloader.__is_remote_changed  sS    w4??#8#88 36Ndoo22236Ndoo222	4rD   c                    | j                   5  | j                  j                  |       | xj                  |j                  z  c_        | j
                  d   j                  |j                  |j                  |j                  |j                  d       | j                  | j
                         d d d        y # 1 sw Y   y xY w)NrT   r   rW   rX   r   )r   r   rR   r   r.   r   r   rW   rX   r   r   )rs   r   s     r2   __finish_partz"_ResumableDownloader.__finish_part  s    [[ 	,!!((.  DII- MM'"))$:J:J48JJ26((7;}}+F G T]]+	, 	, 	,s   B!B77C c                 J    ddj                  d t        d      D              z   S )Nz.tmp- c              3   b   K   | ]'  }t        j                  t        j                         ) y wrw   )randomchoicestringascii_lowercase)r   rV   s     r2   r   z8_ResumableDownloader.__gen_tmp_suffix.<locals>.<genexpr>  s      Z1v/E/E!F Zs   -/   )joinrQ   ry   s    r2   __gen_tmp_suffixz%_ResumableDownloader.__gen_tmp_suffix  s#     ZPUVXPY ZZZZrD   NNNNNNrw   )r   r   r   ru   r>   r   r   r   r   r   r   r   r   r   __classcell__r   s   @r2   r;   r;   X  sQ    #'!W@.'!*#JQ24
	,[rD   r;   c                   t     e Zd ZdZ	 	 	 	 	 	 d fd	Zd ZddZd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Z xZS )r&   uq  以断点续传方式上传文件。

    :param bucket: :class:`Bucket <oss2.Bucket>` 对象
    :param key: 文件名
    :param filename: 待上传的文件名
    :param size: 文件总长度
    :param store: 用来保存进度的持久化存储
    :param headers: 传给 `init_multipart_upload` 的HTTP头部
    :param part_size: 分片大小。优先使用用户提供的值。如果用户没有指定，那么对于新上传，计算出一个合理值；对于老的上传，采用第一个
        分片的大小。
    :param progress_callback: 上传进度回调函数。参见 :ref:`progress_callback` 。
    c                    t         t        |   |||||xs
 t               |       d| _        || _        t        j                  |t        j                        | _	        t        j                  j                  |      | _        t        j                  |	t        j                        | _        d | _        |
| _        t%        j&                         | _        d | _        d| _        d | _        d| _        d| _        d | _        t7        | j8                  t:              rd| _        d| _        t<        j?                  djA                  |jB                  tE        |      | j                  | j                               y )N)r   ResumableUploadr   FTzOInit _ResumableUploader, bucket: {0}, key: {1}, part_size: {2}, num_thread: {3})#r   r&   ru   ResumableStore_ResumableUploader__op_ResumableUploader__headersr	   r$   r   _ResumableUploader__part_sizer!   r"   getmtime_ResumableUploader__mtimemultipart_num_threads_ResumableUploader__num_threads_ResumableUploader__upload_id_ResumableUploader__paramsro   rp   _ResumableUploader__lock_ResumableUploader__record!_ResumableUploader__finished_size"_ResumableUploader__finished_parts_ResumableUploader__encryption)_ResumableUploader__record_upload_context"_ResumableUploader__upload_contextr   r*   r   r   r   r   r    r   )rs   r*   r+   r,   r.   r-   r   r   r   r   r   r   s              r2   ru   z_ResumableUploader.__init__  s0    	 $0h161J.:JCT 	1 	V &	 #<<	83E3EFww''1%\\+x7U7UV  nn&  $!',$ $dkk<0 $D+/D(fmm	#0@0@$BTBTV 	WrD   c                 v   | j                          | j                  | j                        }t        |d       }t        j                  dj                  |             t        t        j                  | j                  |      | j                  g| j                  z        }|j                          | j                  | j                         t!        | j"                  t$        t&        g      }| j(                  j+                  | j,                  | j.                  | j                  |      }| j1                          |S )Nc                     | j                   S rw   r   r   s    r2   r   z+_ResumableUploader.upload.<locals>.<lambda>E  r   rD   r   zParts need to upload: {0})parts_to_uploadr   )_ResumableUploader__load_record'_ResumableUploader__get_parts_to_uploadr$  r   r   r   r   r   r   r   _ResumableUploader__producer_ResumableUploader__consumerr  r   r   r.   r6   r  r7   OSS_OBJECT_ACLr*   complete_multipart_uploadr+   r  rz   )rs   r*  r   r   r0   s        r2   r'   z_ResumableUploader.uploadA  s    44T5J5JK 6MN077HIi''Y'$*<*<<>	dii()$..;Ln:]^66txxAQAQSWShShry6zrD   c                 4    |D ]  }|j                  |        y rw   r   )rs   r   r*  r   s       r2   r   z_ResumableUploader.__producerT  s    # 	DEE$K	rD   c                 N    	 |j                         }|y | j                  |       %rw   )r$   _ResumableUploader__upload_partr   s      r2   r   z_ResumableUploader.__consumerX  s+    557D|t$ rD   c           
         t        t        | j                        d      5 }| j                  | j                         |j                  |j                  t        j                         t        | j                  t        t        g      }| j                  rh| j                  j                  | j                   | j"                  |j$                  t'        j(                  ||j*                        || j,                        }n\| j                  j                  | j                   | j"                  |j$                  t'        j(                  ||j*                        |      }t.        j1                  dj3                  |j$                  |j4                  |j*                               | j7                  t9        |j$                  |j4                  |j*                  |j:                               d d d        y # 1 sw Y   y xY w)Nr   )r   upload_contextr+  zTUpload part success, add part info to record, part_number: {0}, etag: {1}, size: {2})r.   r   )r(   r   r,   r   r#  r   rW   r!   r   r6   r  r7   r8   r%  r*   upload_partr+   r  r   r   SizedFileAdapterr.   r'  r   r   r   r   _ResumableUploader__finish_partr   crcr   s        r2   __upload_partz _ResumableUploader.__upload_part`  sr   *T]]+T2 	ma!!$"6"67FF4::r{{+-dnn?PRc>deG  004;K;KTM]M]161G1G4991U_f@D@U@U 1 W 004;K;KTM]M]161G1G4991U_f 1 h LLovv  &++tyy: ;x(8(8&++DII`f`j`jkl	m 	m 	ms   G G))G2c                     | j                   5  | j                  j                  |       | xj                  |j                  z  c_        d d d        y # 1 sw Y   y xY wrw   )r!  r$  rR   r#  r.   )rs   	part_infos     r2   r  z _ResumableUploader.__finish_partr  sG    [[ 	3!!((3  INN2 	3 	3 	3s   ;AAc           	      4
   | j                         }t        j                  dj                  |             |r8| j	                  |      s't        j                  d       | j                          d }|rQ| j                  |      r@t        j                  dj                  | j                               | j                          d }|rM| j                  |d         s9t        j                  dj                  |d                | j                          d }|s?t        | j                  t        j                  g      }t        | j                  | j                         }t        j                  dj                  | j                  | j                   |             | j"                  rt%        j&                  | j                  |      }| j(                  j+                  | j,                  | j.                  ||      j0                  }| j2                  r|j4                  }|j6                  |j8                  t;        |j<                        t;        |j>                        |j@                  d}n;| j(                  j+                  | j,                  | j.                  |      j0                  }| jB                  || jD                  | j                  | jF                  | j(                  jH                  | j,                  |d}| j2                  r|d	<   t        j                  d
j                  | j(                  jH                  | j,                  ||             | jK                  |       || _&        | jL                  d   | _        | jL                  d   | _'        | j2                  r.d	| jL                  v r| jL                  d	   }|d   }|d   }	|	| j(                  jP                  jR                  jT                  k7  s#|| j(                  jP                  j6                  k7  rd}
tW        jX                  |
|       t%        jZ                  | j(                  jP                  jR                  |d   t]        |d         t]        |d         |d         }t%        j&                  | j                  | j                   |      | _/        n>d}
tW        jX                  |
|       d	| jL                  v rd}
tW        j`                  |
|       | jc                         | _2        tg        d | jd                  D              | _4        y )Nr   r   z-File: {0} has been changed, delete the record	upload_idz7Multipart upload: {0} does not exist, delete the recordzMUpload File size: {0}, User-specify part_size: {1}, Calculated part_size: {2})wrap_algcek_algencrypted_keyencrypted_ivmat_desc)r   r?  r   r.   r   r*   r+   r   content_crypto_materialzEAdd new record, bucket: {0}, key: {1}, upload_id: {2}, part_size: {3}r   r@  rA  z@Envelope or data encryption/decryption algorithm is inconsistentrB  rC  rD  zUIf record_upload_context flag is true, content_crypto_material must in the the recordzWcontent_crypto_material must in the the record, but record_upload_context flat is falsec              3   4   K   | ]  }|j                     y wrw   r   r   s     r2   r   z3_ResumableUploader.__load_record.<locals>.<genexpr>  r   r   )5r   r   r   r   "_ResumableUploader__is_record_saner   rz    _ResumableUploader__file_changedr,   !_ResumableUploader__upload_existsrf   r   r   
SEQUENTIALrJ   r.   r  r%  r   MultipartUploadCryptoContextr*   init_multipart_uploadr+   r  r?  r&  rE  r@  rA  r   rB  rC  rD  r  rk   r  r    r   r"  r  crypto_providercipheralgr   InconsistentErrorContentCryptoMaterialr   r'  InvalidEncryptionRequest&_ResumableUploader__get_finished_partsr$  r   r#  )rs   r~   r   r   r6  r?  materialmaterial_recordr@  rA  err_msgrE  s               r2   r   z _ResumableUploader.__load_recordw  s;   !!#-44V<=$//7KKMNFd))&1KKGNNt}}]^F$..vk/BCKKQXXY_`kYlmnF+DMMF<M<M;NOF+DIIt7G7GHILLhoo		4++Y8 9  !'!D!DTYYPY!Z KK==dhhX^>LNNWi //-EEH3;3D3DQYQaQa8KHLbLb8c7J8K`K`7a3;3D3D'FO
 !KK==dhhX^_ii	!%QUQ^Q^hlhqhq#||t{{7N7NW[W_W_nwyF ++4C01LL`gg''9iI J V$==5==5''(DMM9"&--0I"J*:6))4dkk99@@DDDTXT_T_ToToTxTxHx`G$66wEE*0*F*Ft{{GbGbGiGiGVWaGbG\KZ[jKkHmG\KZ[iKjHlGVWaGb+d' )/(K(KDIIW[WgWgLc)e% r 227DAA )DMM9s 99'4HH $ 9 9 ;""I43H3H"IIrD   c                     g }t        | j                  t        t        g      }t	        | j
                  | j                  | j                  |      D ]  }|j                  |        |S )Nr+  )	rb   r  OSS_SERVER_SIDE_ENCRYPTIONOSS_SERVER_SIDE_DATA_ENCRYPTIONr   r*   r+   r  rR   )rs   rT   rA   r   s       r2   __get_finished_partsz'_ResumableUploader.__get_finished_parts  s_    /+-LMO !dhh8H8HR_` 	DLL	 rD   c           
          	 t        | j                  t        t        g      }t	        t        j                  | j                  | j                  |dd|             y# t        j                  $ r Y yw xY w)N0r   )	max_partsr   TF)rb   r  rX  rY  r   r   r   r*   r+   r   NoSuchUpload)rs   r?  rA   s      r2   __upload_existsz"_ResumableUploader.__upload_exists  sg    	3DNN13RSUM''TXXy#YZdqrs  && 		s   AA A10A1c                 N    |d   | j                   k7  xs |d   | j                  k7  S )Nr   r.   )r  r.   r}   s     r2   __file_changedz!_ResumableUploader.__file_changed  s'    g$,,.M&.DII2MMrD   c                     t        | j                  | j                        }|s|S t        d |D              }|D ]  }|j                  |v s||j                  =   |j                         S )Nc              3   8   K   | ]  }|j                   |f  y wrw   r   r   s     r2   r   z;_ResumableUploader.__get_parts_to_upload.<locals>.<genexpr>  s     CAammQ/Cs   )rY   r.   r  rd   r   values)rs   parts_uploaded	all_partsall_parts_mapuploadeds        r2   __get_parts_to_uploadz(_ResumableUploader.__get_parts_to_upload  sm    #DIIt/?/?@	CCC& 	8H##}4!("6"67	8 ##%%rD   c                    	 |d   | j                   k7  r(t        j                  dj                  |d                ydD ]?  }t	        ||   t
              rt        j                  dj                  |||                 y dD ]?  }t	        ||   t              rt        j                  dj                  |||                 y t	        |d   t              s;t	        |d   t              s(t        j                  d	j                  |d                yy# t        $ r8}t        j                  d
j                  |j                               Y d }~yd }~ww xY w)Nr   r   F)r?  r   r*   r+   z3Type Error, {0} in record is not a string type: {1})r.   r   z5Type Error, {0} in record is not an integer type: {1}r   zBType Error, mtime in record is not a float or an integer type: {0}r   T)
r  r   r   r   r   r   r   floatr   r   r   s       r2   r  z#_ResumableUploader.__is_record_sane  s9   	i DII-PWWX^_hXijkB !!&+s3LL!V!]!]^acijmcn!op !
 - !!&+s3LL!X!_!_`cekloep!qr !
 fWos3JvgX]<^X__`fgn`opr 	  	LL-44QVV<=	s5   9D D )D ?D )D AD 	E.EEr  rw   )r   r   r   __doc__ru   r'   r.  r/  r4  r9  r,  rS  rI  rH  r-  rG  r  r  s   @r2   r&   r&     s_     #'!&WP&%m$3
OJb	N&rD   r&   z.py-oss-uploadz.py-oss-downloadc                   *    e Zd Zd Zd Zd Zd Zd Zy)_ResumableStoreBasec                 *   t         j                  dj                  ||             t        j                  j                  ||      | _        t        j                  j                  | j                        ry t        j                  | j                         y )Nz6Init ResumableStoreBase, root path: {0}, temp dir: {1})
r   r   r   r!   r"   r  dirisdirr   	makedir_p)rs   rootrp  s      r2   ru   z_ResumableStoreBase.__init__  s[    MTTUY[^_`77<<c*77=="!rD   c                    | j                  |      }t        j                  dj                  ||             t        j
                  j                  |      s%t        j                  dj                  |             y 	 t        t        |      d      5 }t        j                  |      }d d d        t              S # 1 sw Y   xY w# t        $ r t	        j                  |       Y y w xY w)Nz4ResumableStoreBase: get key: {0} from file path: {1}zfile {0} is not existr)_ResumableStoreBase__pathr   r   r   r!   r"   r   r(   r   r   loadr   
ValueErrorremove)rs   r+   pathnamer1   contents        r2   r$   z_ResumableStoreBase.get  s    ;;s#KRRSVX`abww~~h'LL077AB
	&j*C0 'A))A,' W%%' ' 	IIh	s*   <C B:'C :C?C C'&C'c                     | j                  |      }t        t        |      d      5 }t        j                  ||       d d d        t
        j                  dj                  |||             y # 1 sw Y   0xY w)Nwz>ResumableStoreBase: put key: {0} to file path: {1}, value: {2})rv  r(   r   r   dumpr   r   r   )rs   r+   valuerz  r1   s        r2   r|   z_ResumableStoreBase.put,  se    ;;s#*X&, 	 IIeQ	  	U\\]`bjlqrs	  	 s   A--A6c                     | j                  |      }t        j                  |       t        j	                  dj                  ||             y )Nz3ResumableStoreBase: delete key: {0}, file path: {1})rv  r!   ry  r   r   r   )rs   r+   rz  s      r2   rx   z_ResumableStoreBase.delete4  s8    ;;s#
		(JQQRUW_`arD   c                 V    t         j                  j                  | j                  |      S rw   )r!   r"   r  rp  )rs   r+   s     r2   __pathz_ResumableStoreBase.__path:  s    ww||DHHc**rD   N)r   r   r   ru   r$   r|   rx   rv  r   rD   r2   rn  rn    s    "&*tb+rD   rn  c                 z    t         j                  j                  t         j                  j                  |             S rw   )r!   r"   normpathnormcase)r"   s    r2   _normalize_pathr  >  s&    77BGG,,T233rD   c                   4     e Zd ZdZd fd	Zed        Z xZS )r  u   保存断点上传断点信息的类。

    每次上传的信息会保存在 `root/dir/` 下面的某个文件里。

    :param str root: 父目录，缺省为HOME
    :param str dir: 子目录，缺省为 `_UPLOAD_TEMP_DIR`
    c                     t         t        |   |xs t        j                  j                  d      |xs t               y N~)r   r  ru   r!   r"   
expanduser_UPLOAD_TEMP_DIRrs   rs  rp  r   s      r2   ru   zResumableStore.__init__K  s.    nd,T-LRWW5G5G5LcNeUefrD   c                     t        |      }dj                  | |      }t        j                  |      dz   t        j                  |      z   S )Noss://{0}/{1}--r  r   r   
md5_string)r    r+   r,   filepathoss_pathnames        r2   rm   zResumableStore.make_store_keyN  sD    "8,&--k3?-4u7G7G7QQQrD   r   r   r   r   rl  ru   r   rm   r  r  s   @r2   r  r  B  s$    g R RrD   r  c                   6     e Zd ZdZd fd	Zedd       Z xZS )r   u   保存断点下载断点信息的类。

    每次下载的断点信息会保存在 `root/dir/` 下面的某个文件里。

    :param str root: 父目录，缺省为HOME
    :param str dir: 子目录，缺省为 `_DOWNLOAD_TEMP_DIR`
    c                     t         t        |   |xs t        j                  j                  d      |xs t               y r  )r   r   ru   r!   r"   r  _DOWNLOAD_TEMP_DIRr  s      r2   ru   zResumableDownloadStore.__init___  s1    $d4T5TRWW=O=OPS=TVYVo]oprD   c                     t        |      }|dj                  | |      }ndj                  | ||      }t        j                  |      dz   t        j                  |      z   S )Nr  zoss://{0}/{1}?versionid={2}r  r  )r    r+   r,   
version_idr  r  s         r2   rm   z%ResumableDownloadStore.make_store_keyb  s_    "8,*11+sCL8??SR\]L-4u7G7G7QQQrD   r   rw   r  r  s   @r2   r   r   V  s$    q R RrD   r   c                     t        | |      S Nrs  rp  )r  r  s     r2   make_upload_storer  m  s    t--rD   c                     t        | |      S r  )r   r  s     r2   make_download_storer  q  s    !t55rD   c                   @    e Zd ZddZed        Zd Zd Zed        Zy)rS   Nc                 <    || _         || _        || _        || _        y rw   r  )rs   r   rW   rX   r   s        r2   ru   z_PartToProcess.__init__v  s    &
 rD   c                 4    | j                   | j                  z
  S rw   )rX   rW   ry   s    r2   r.   z_PartToProcess.size|  s    xx$**$$rD   c                 ,    t        | j                        S rw   )hash_PartToProcess__keyry   s    r2   __hash__z_PartToProcess.__hash__  s    DJJrD   c                 4    | j                   |j                   k(  S rw   )r  )rs   others     r2   __eq__z_PartToProcess.__eq__  s    zzU[[((rD   c                 H    | j                   | j                  | j                  fS rw   )r   rW   rX   ry   s    r2   __keyz_PartToProcess.__key  s    TXX55rD   rw   )	r   r   r   ru   propertyr.   r  r  r  r   rD   r2   rS   rS   u  s9    ! % % ) 6 6rD   rS   )NNNNNNNrw   r   )7rl  r!   r  r   r   r   r   r   r	   r
   r   crypto_bucketr   r   r   r   compatr   r   r   r   
task_queuer   r   r   ro   r
  r  logging	getLoggerr   r   r3   rC   r   rJ   rF   rY   r6   rb   rf   objectrh   r<   r;   r&   r  r  rn  r  r  r   r  r  rS   r   rD   r2   <module>r     sT   
  =      '  #  : : !      			8	$  !)-#'+!% =B +/!%)-#'!"#?9D "  (,^ 462#C& #CL&  p[. p[f{, {| $ ' .+& .+b4R( R(R0 R..66V 6rD   