File manager - Edit - /home/u478019808/domains/bestandroidphones.store/public_html/static/img/logo/commands.tar
Back
__pycache__/defacl.cpython-39.pyc 0000644 00000026404 15025166704 0012637 0 ustar 00 a +(Wg : � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z dd l m Z dd l m Z dd l m Z dd l mZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) dd l*m+Z+ d Z,d Z-d Z.d Z/d!Z0d"Z1e,e-�2d#� e.�2d#� d$ Z3d%d#�4e/e0 e1 g� Z5ee3e5�Z6ee-e0�Z7ee,e/�Z8ee.e1�Z9G d&d'� d'e�Z:dS )(zFImplementation of default object acl command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)� gcs_json_api)�metrics)�AccessDeniedException)�BadRequestException)� Preconditions)�ServiceException)�Command)�SetAclExceptionHandler)�SetAclFuncWrapper)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)� acl_helper)�NO_MAX)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�PRIVATE_DEFAULT_OBJ_ACLzJ gsutil defacl set (<file-path>|<predefined-acl>) gs://<bucket_name>... z( gsutil defacl get gs://<bucket_name> zF gsutil defacl ch [-f] -u|-g|-d|-p <grant>... gs://<bucket_name>... a[ <B>SET</B> The ``defacl set`` command sets default object ACLs for the specified buckets. If you specify a default object ACL for a certain bucket, Cloud Storage applies the default object ACL to all new objects uploaded to that bucket, unless an ACL for that object is separately specified during upload. Similar to the ``acl set`` command, the ``defacl set`` command specifies either a predefined ACL or the path to a file that contains ACL text. See "gsutil help acl" for examples of editing and setting ACLs via the acl command. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. Setting a default object ACL on a bucket provides a convenient way to ensure newly uploaded objects have a specific ACL. If you don't set the bucket's default object ACL, it will default to project-private. If you then upload objects that need a different ACL, you will need to perform a separate ACL update operation for each object. Depending on how many objects require updates, this could be very time-consuming. z{ <B>GET</B> Gets the default ACL text for a bucket, which you can save and edit for use with the "defacl set" command. a� <B>CH</B> The "defacl ch" (or "defacl change") command updates the default object access control list for a bucket. The syntax is shared with the "acl ch" command, so see the "CH" section of "gsutil help acl" for the full help description. <B>CH EXAMPLES</B> Grant anyone on the internet READ access by default to any object created in the bucket example-bucket: gsutil defacl ch -u AllUsers:R gs://example-bucket NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Add the user john.doe@example.com to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -u john.doe@example.com:READ gs://example-bucket Add the group admins@example.com to the default object ACL on bucket example-bucket with OWNER access: gsutil defacl ch -g admins@example.com:O gs://example-bucket Remove the group admins@example.com from the default object ACL on bucket example-bucket: gsutil defacl ch -d admins@example.com gs://example-bucket Add the owners of project example-project-123 to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -p owners-example-project-123:R gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -u Add or modify a user entity's role. � z z. The defacl command has three sub-commands: c s� e Zd ZdZejdg d�ededdddej ej gej e � � e � � ge �d�ge � � gd�d � Zejdg d �d d eeeed �d�Z� fdd�Zdd� Zdd� Zdd� Zdd� Zee ddd�dd � �Z d d � Z d d � Z � Z S )!� DefAclCommandz(Implementation of gsutil defacl command.�defacl)� setdefacl� getdefacl�chdefacl� z fg:u:d:p:F� )�set�get�ch) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)z default aclr r r! � command_helpz*Get, set, or change default ACL on buckets)r% r$ r& )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� | j �d�}|dkr&tg d�i d�}n�|dkr�| j �d�}tj�|�r`tdddd | gi d�}q�|tjv rvtj| }n|}tdddd | gi d�}nN|d kr�| �� t � | j �| _ tg d �t d �t d �t d �t d�t d�d�d�}t � �|�S )Nr r% )�storage�buckets�describez,--format=multi(defaultObjectAcl:format=json)z--raw)�gcloud_command�flag_mapr$ r9 r: �updatez--default-object-acl-file=z --predefined-default-object-acl=r& )r9 r: r> z --add-default-object-acl-grantz!--remove-default-object-acl-grantz--continue-on-error)�-g�-p�-u�-dz-f)�args�popr �os�path�isfiler �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION� ParseSubOptsr �translate_sub_opts_for_shim�sub_optsr �super�get_gcloud_storage_args)�self� sub_command�gcloud_storage_map�acl_file_or_predefined_acl�predefined_acl�� __class__� �$/opt/gsutil/gslib/commands/defacl.pyrM � sT � ��������� z%DefAclCommand.get_gcloud_storage_argsc C s6 | j s| �� | j d �� dks*| jdkr.dS dS d S )Nr r$ r r# )rC �$RaiseWrongNumberOfArgumentsException�lower�command_alias_used�rN rU rU rV �_CalculateUrlsStartArg� s �z$DefAclCommand._CalculateUrlsStartArgc C sR t | jd ��� s td| j ��z| �tt� W n tyL | � � � Y n0 d S )N����)URL must name a bucket for the %s command) r rC �IsBucketr � command_name�SetAclCommandHelperr r r �_WarnServiceAccountsrZ rU rU rV � _SetDefAcl s � zDefAclCommand._SetDefAclc C s4 t | jd ��� s td| j ��| �| jd � d S )Nr r] )r rC r^ r r_ �GetAndPrintAclrZ rU rU rV � _GetDefAcl s �zDefAclCommand._GetDefAclc C s: d| _ g | _| jr�| jD ]�\}}|dkrB| j�tj|tjjd�� |dkrd| j�tj|tjjd�� |dkr�| j�tj|tjj d�� |dkr| j�t� |�� q| js�t d��t | j �r�t| j d �jd kr�t d �| j���t� }| j D ]4}| �|�D ]$}|j�� �st d ��|�|j� q�q�|D ]}| �|� �q$d S ) zDParses options and changes default object ACLs on specified buckets.Tr? )� scope_typerA r@ rB zFPlease specify at least one access change with the -g, -u, or -d flagsr �gsz2The "{0}" command can only be used with gs:// URLsz5The defacl ch command can only be applied to buckets.N)�parse_versions�changesrK �appendr � AclChange� ChangeType�GROUP�USER�PROJECT�AclDelr r rC r �scheme�formatr_ r$ �WildcardIterator� storage_urlr^ �add�ApplyAclChanges)rN �o�aZ bucket_urlsZurl_arg�resultrs rU rU rV � _ChDefAcl sL ��� ��� �zDefAclCommand._ChDefAcl� )�tries� timeout_secsc C s� | j j|j|jddgd�}|j}| �||�dkrB| j�d|� dS |sP|�t � zFt |j d�}t j |d�}| j j|j|||jd gd � | j�d |� W nX ty� } z td t|� ��W Y d}~n.d}~0 ty� | �� td | ��Y n0 dS )z8Applies the changes in self.changes to the provided URL.�defaultObjectAcl�metageneration)�provider�fieldsr zNo changes to %sN)�meta_gen_match)r} �id)� preconditionsr r� zUpdated default ACL on %sz$Received bad request from server: %szTFailed to set acl for %s. Please ensure you have OWNER-role access to this resource.)� gsutil_api� GetBucket� bucket_namerp r} �$_ApplyAclChangesAndReturnChangeCount�logger�infori r r r~ �apitools_messages�Bucket� PatchBucketr r �strr ra )rN �url�bucket� current_aclr� �bucket_metadata�erU rU rV ru = s8 � �$ �z DefAclCommand.ApplyAclChangesc C s* d}| j D ]}||�||d| j�7 }q |S )Nr r )rh �Executer� )rN rs Zdefacl_message�modification_count�changerU rU rV r� e s �z2DefAclCommand._ApplyAclChangesAndReturnChangeCountc C s� | j �d�}| jdd� d| _d| _|dkr4| j}n2|dkrD| j}n"|dv rT| j}ntd|| j f ��t j |g| j d � |� dS ) z+Command entry point for the defacl command.r T)� check_argsFr% r$ )r& r� zEInvalid subcommand "%s" for the %s command. See "gsutil help defacl".)� subcommandsrK ) rC rD rI �def_acl�continue_on_errorrd rb ry r r_ r �LogCommandParamsrK )rN �action_subcommand�funcrU rU rV � RunCommandl s$ ��zDefAclCommand.RunCommand)!�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeFileURLOrCannedACLArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrM r[ rb rd ry r r ru r� r� � __classcell__rU rU rS rV r � sP � ���� 5 ( 'r );r� � __future__r r r r rE �gslibr r �gslib.cloud_apir r r r � gslib.commandr r r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_SET_DESCRIPTION�_GET_DESCRIPTION�_CH_DESCRIPTION�lstripr� �join� _DESCRIPTIONr� r� r� r� r rU rU rU rV �<module> s\ ; ��� __pycache__/kms.cpython-39.pyc 0000644 00000035606 15025166704 0012217 0 ustar 00 a +(Wg�R � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl)m+Z+ dZ,dZ-d Z.e,e-�/d � e.�/d � d Z0d Z1d Z2d Z3d!e1e2 e3 Z4ee0e4�Z5ee,e1�Z6ee-e2�Z7ee.e3�Z8e+d"d#ge*d$�e*d%�d&�d'�Z9e+d"d#gd(e*d$�id'�Z:G d)d*� d*e�Z;dS )+z/This module provides the kms command to gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�AccessDeniedException)�ServiceException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�KmsApi)�PopulateProjectId)�Binding)�storage_v1_messages)� text_util)�NO_MAX)� ValidateCMEK)�Retry)�GcloudStorageFlag)�GcloudStorageMapz4 gsutil kms authorize [-p <proj_id>] -k <kms_key> zJ gsutil kms encryption [(-d|[-k <kms_key>])] [-w] gs://<bucket_name>... z, gsutil kms serviceaccount [-p <proj_id>] � a� <B>AUTHORIZE</B> The authorize sub-command checks that the default (or supplied) project has a Cloud Storage service agent created for it, and if not, it creates one. It then adds appropriate encrypt/decrypt permissions to Cloud KMS resources such that the service agent can write and read Cloud KMS-encrypted objects in buckets associated with the service agent's project. <B>AUTHORIZE EXAMPLES</B> Authorize "my-project" to use a Cloud KMS key: gsutil kms authorize -p my-project \ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key <B>AUTHORIZE OPTIONS</B> -k <key> The path to the KMS key to use. The path has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -p <project> The ID or number of the project being authorized to use the Cloud KMS key. If this flag is not included, your default project is authorized. a� <B>ENCRYPTION</B> The encryption sub-command is used to set, display, or clear a bucket's default KMS key, which is used to encrypt newly-written objects if no other key is specified. <B>ENCRYPTION EXAMPLES</B> Set the default KMS key for my-bucket: gsutil kms encryption \ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key \ gs://my-bucket Show the default KMS key for my-bucket, if one is set: gsutil kms encryption gs://my-bucket Clear the default KMS key so newly-written objects are not encrypted using it: gsutil kms encryption -d gs://my-bucket Once you clear the default KMS key, newly-written objects are encrypted with Google-managed encryption keys by default. <B>ENCRYPTION OPTIONS</B> -k <key> Set the default KMS key for my-bucket using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -w (used with -k key) Display a warning rather than failing if gsutil is unable to verify that the specified key contains the correct IAM bindings for encryption/decryption. This is useful for users that do not have getIamPolicy permission but know that the key has the correct IAM policy for encryption in the user's project. -d Clear the default KMS key. a" <B>SERVICEACCOUNT</B> The serviceaccount sub-command displays the Cloud Storage service agent that is used to perform Cloud KMS operations against your default project (or a supplied project). <B>SERVICEACCOUNT EXAMPLES</B> Show the service account for my-project: gsutil kms serviceaccount -p my-project <B>SERVICEACCOUNT OPTIONS</B> -p <project> The ID or number of the project whose Cloud Storage service agent is being requested. If this flag is not included, your default project is used. aS The kms command is used to configure Google Cloud Storage and Cloud KMS resources to support encryption of Cloud Storage objects with `Cloud KMS keys <https://cloud.google.com/storage/docs/encryption/customer-managed-keys>`_. The kms command has three sub-commands that deal with configuring Cloud Storage's integration with Cloud KMS: ``authorize``, ``encryption``, and ``serviceaccount``. Before using this command, read the `prerequisites <https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#prereqs>`_. for using Cloud KMS with Cloud Storage. �storagez service-agentz --projectz--authorize-cmek)�-p�-k��gcloud_command�flag_mapr c s� e Zd ZdZejdededdddej gej g e � d�gg d�d� Z ej dg dd eeeed�d �Zeeed �i d �Z� fd d�Zdd� Zeeddd�dd� �Zdd� Z dd� Z dd� Z dd � Z d d � Z d d � Z!d!d"� Z"d#d"� Z"� Z#S )$� KmsCommandz!Implements of gsutil kms command.�kms� zdk:p:wF�� authorize� encryption�serviceaccount) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz Configure Cloud KMS encryption)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)r$ r& r c s� | j d dkr�tdtddgtd�td�td�d�d �ii d �}d | j v sRd | j v rj|jd jd g7 _q�|jd jg d �7 _ntj}t� �|�S )Nr r% r �bucketsz --clear-default-encryption-keyz--default-encryption-key� )�-dr �-wr r: r �update)�describezi--format=value[separator=": "](name, encryption.defaultKmsKeyName.yesno(no="No default encryption key."))z--raw)�argsr r r r �gcloud_storage_map�super�get_gcloud_storage_args)�selfr? �� __class__� �!/opt/gsutil/gslib/commands/kms.pyrA � s( ��� � � z"KmsCommand.get_gcloud_storage_argsc C s� | � � d| _d | _d| _| jrx| jD ]P\}}|dkr>|| _q&|dkrX|| _t| j� q&|dkrhd| _q&|dkr&d| _q&| jr�| jdks�| js�td� t � d ����| js�t d �| _d S ) NFr r r: Tr; r% r zoThe "-w" option should only be specified for the "encryption" subcommand and must be used with the "-k" option.) �CheckArguments� clear_kms_key�kms_key� warn_on_key_authorize_failure�sub_opts� project_idr �subcommand_namer �join�textwrap�wrapr )rB rM �o�arE rE rF �_GatherSubOptions s6 ����z KmsCommand._GatherSubOptions� )�tries� timeout_secsc C s� | j j|dd�j}t| jd�}| j�d|� z^|�|�}| j�d|� d}tdd| gd �}||jvr�|j� |� |� ||� d }||fW S t y� | j r�t �d �t�d ��� |df Y S � Y n0 d S )a� Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. �gs��provider)�loggerzGetting IAM policy for %szCurrent policy is %sFz*roles/cloudkms.cryptoKeyEncrypterDecrypterzserviceAccount:%s)�role�membersTr a Warning: Check that your Cloud Platform project's service account has the "cloudkms.cryptoKeyEncrypterDecrypter" role for the specified key. Without this role, you may not be able to encrypt or decrypt objects using the key which will prevent you from uploading or downloading objects.N)� gsutil_api�GetProjectServiceAccount� email_addressr rZ �debugZGetKeyIamPolicyr �bindings�appendZSetKeyIamPolicyr rJ r � print_to_fdrN rO rP )rB rL rI �service_accountZkms_api�policyZadded_new_binding�bindingrE rE rF �_AuthorizeProject s4 � � �� z KmsCommand._AuthorizeProjectc C sj | � d� | js$td| j| jf ��| �| j| j�\}}|rRtd| j| jf � ntd| j| jf � dS )Nr$ z,%s %s requires a key to be specified with -kz9Authorized project %s to encrypt and decrypt with key: %szFProject %s was already authorized to encrypt and decrypt with key: %s.r )rS rI r � command_namerM rg rL �print)rB �_�newly_authorizedrE rE rF � _AuthorizeS s � � �zKmsCommand._Authorizec C s@ t j�� |_tdt|��d� � | jj|j |dg|j d� dS )z�Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. z)Clearing default encryption key for %s...�/r% ��fieldsrY N) �apitools_messages�Bucket�EncryptionValuer% ri �str�rstripr] � PatchBucket� bucket_name�scheme)rB �bucket_metadata� bucket_urlrE rE rF �_EncryptionClearKeyb s � �z KmsCommand._EncryptionClearKeyc C s� |j }z|| d }}W n, tyD | �|| j�\}}|||< Y n0 |r^t�d|| jf � tjj| jd�|_ t dt |�� d� � | j j|j|dg|jd� dS ) aa Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. Fz,Authorized service account %s to use key: %s)�defaultKmsKeyNamez(Setting default KMS key for bucket %s...rm r% rn N)� projectNumber�KeyErrorrg rI r rc rp rq rr r% ri rs rt r] ru rv rw )rB rx ry �svc_acct_for_project_numZbucket_project_numberrd rk rE rE rF �_EncryptionSetKeyq s0 � ��� � �z KmsCommand._EncryptionSetKeyc st � � d� i �� �fdd�}d}� j}|s2� �� |D ]$}� �|�}|D ]}d}||� qHq6|spttt|� ��dS )Nr% c s� | j }|jdkr td� j ��� jj|jddg|jd�}� jrN� �||� dS � j rf� � ||�� dS t |�� d�}|j r�|j jr�td||j jf � n td | � dS ) z6Set, clear, or get the defaultKmsKeyName for a bucket.rW z7The %s command can only be used with gs:// bucket URLs.r% r| rn r rm z!Default encryption key for %s: %sz'Bucket %s has no default encryption key)� storage_urlrw r rh r] � GetBucketrv rH rz rI r rs rt r% r{ ri )�blrry rx �bucket_url_string�rB r~ rE rF �_EncryptionForBucket� s: ��� �� � z4KmsCommand._Encryption.<locals>._EncryptionForBucketFTr )rS r> �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr r �list)rB r� � some_matched�url_args�url_str� bucket_iter�bucket_listing_refrE r� rF � _Encryption� s % zKmsCommand._Encryptionc C sz | � � | jsdg| _| jr:| jD ]\}}|dkr"|| _q"| jsJtd �| _| j�d| j� | jj| jdd�j }t |� dS )Nzgs://r z'Checking service account for project %srW rX r ) rG r> rK rL r rZ r` r] r^ r_ ri )rB rQ rR rd rE rE rF �_ServiceAccount� s" ��zKmsCommand._ServiceAccountc C sT z2t � | j| jj�\| _| _tj| jd� || �W S t jyN | �� Y n0 d S )N)rK ) �getoptr> � command_specr* rK r �LogCommandParams� GetoptError� RaiseInvalidArgumentException)rB �funcrE rE rF �_RunSubCommand� s � zKmsCommand._RunSubCommandc C s2 | j jdd�tjkr.td�t�d| j ����dS )�(Command entry point for the kms command.rW rX r z�The "%s" command can only be used with the GCS JSON API. If you have only supplied hmac credentials in your boto file, please instead supply a credential type that can be used with the JSON API.N) r] �GetApiSelectorr �JSONr rN rO rP rh )rB rE rE rF � RunCommand� s ���zKmsCommand.RunCommandc C s� | j jdd�tjkr.td�t�d| j ����t j t j t j d�}| j �d�| _| j|v rxtj| jgd� | �|| j �S td| j| jf ��d S ) r� rW rX r z�The "%s" command can only be used with the GCS JSON API, which cannot use HMAC credentials. Please supply a credential type that is compatible with the JSON API (e.g. OAuth2) in your boto config file.r# r )� subcommandsz+Invalid subcommand "%s" for the %s command.N)r] r� r r� r rN rO rP rh r rl r� r� r> �poprM r r� r� )rB Zmethod_for_subcommandrE rE rF r� � s& ���� �)$�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r r� r � MakeNCloudBucketURLsArgumentr� �HelpSpec�_DETAILED_HELP_TEXT�_authorize_help_text�_encryption_help_text�_serviceaccount_help_text� help_specr �_AUTHORIZE_COMMAND�_SERVICEACCOUNT_COMMANDr? rA rS r r rg rl rz r r� r� r� r� � __classcell__rE rE rC rF r � s\ ���� �� 4!; r )<r� � __future__r r r r r� rO �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr Z gslib.kms_apir �gslib.project_idr Z3gslib.third_party.kms_apitools.cloudkms_v1_messagesr �"gslib.third_party.storage_apitoolsr rp Z gslib.utilsr �gslib.utils.constantsr � gslib.utils.encryption_helperr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Z_AUTHORIZE_SYNOPSISZ_ENCRYPTION_SYNOPSISZ_SERVICEACCOUNT_SYNOPSIS�lstripr� Z_AUTHORIZE_DESCRIPTIONZ_ENCRYPTION_DESCRIPTIONZ_SERVICEACCOUNT_DESCRIPTION� _DESCRIPTIONr� r� r� r� r� r� r rE rE rE rF �<module> s~ ��+ �� ������� __pycache__/compose.cpython-39.pyc 0000644 00000011111 15025166704 0013053 0 ustar 00 a +(Wg" � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dZ dZ de de Z!G dd� de �Z"dS )z;Implementation of compose command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literals)�config)�BucketListingObject)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)�storage_v1_messages)�GetEncryptionKeyWrapper)�GcloudStorageFlag)�GcloudStorageMap)�PreconditionsFromHeaders� zb gsutil compose gs://bucket/source_obj1 [gs://bucket/source_obj2 ...] gs://bucket/composite_obj z <B>SYNOPSIS</B> aH <B>DESCRIPTION</B> The compose command creates a new object whose content is the concatenation of a given sequence of source objects under the same bucket. gsutil uses the content type of the first source object to determine the destination object's content type and does not modify or delete the source objects as part of the compose command. For more information, see the `composite objects topic <https://cloud.google.com/storage/docs/composite-objects>`_. There is a limit (currently %d) to the number of components that can be composed in a single operation. c @ s� e Zd ZdZejddgeded ddddej ej gej e � � gd� Z ejddgdd ei d �Zeg d �i d �Zd d� Zdd� ZdS )�ComposeCommandz)Implementation of gsutil compose command.�compose�concat� � F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz>Concatenate a sequence of objects into a new composite object.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objectsr )�gcloud_command�flag_mapc C s |j dkrtdt|� ��d S )N�gsz7"compose" called on URL with unsupported provider (%s).)�schemer �str)�self�url� r4 �%/opt/gsutil/gslib/commands/compose.py� CheckProvider_ s �z ComposeCommand.CheckProviderc C s� | j d }| j dd� | _ t|�}| �|� |�� r@td| ��tj|j|jd�}g }d}| j D ]�}t |�r|| � |�� � }nt t|��g}|D ]v}|j } | �| � | j|jkr�td��|s�| }tjj| jd�} | �� r�t| j�| _|�| � t|�tkr�tdt ��q�q`|�std��| jj|j|j|jd d gd �} | j|_| j|_t| j�pPi �} | j � d |t|�� | jj ||| |jt t �d � dS )z,Command entry point for the compose command.���NzQA version-specific URL (%s) cannot be the destination for gsutil compose - abort.)�name�bucketz,GCS does not support inter-bucket composing.)r8 z>"compose" called with too many component objects. Limit is %d.z/"compose" requires at least 1 component object.�contentEncoding� contentType)�provider�fieldsz)Composing %s from %d component object(s).)� preconditionsr< �encryption_tuple)!�argsr r6 � HasGenerationr �apitools_messages�Object� object_name� bucket_namer �WildcardIterator� IterObjectsr � storage_url�ComposeRequest�SourceObjectsValueListEntry�int� generation�append�len�MAX_COMPOSE_ARITY� gsutil_api�GetObjectMetadatar0 r; r: r �headers�logger�info� ComposeObjectr r ) r2 Ztarget_url_strZ target_url�dst_obj_metadata� componentsZ first_src_url� src_url_strZ src_url_iter�blr�src_url�src_obj_metadataZfirst_src_obj_metadatar> r4 r4 r5 � RunCommande sp �� �� � � ��zComposeCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISrO r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr �gcloud_storage_mapr6 r\ r4 r4 r4 r5 r = s: �� �r N)#r` � __future__r r r r �botor �gslib.bucket_listing_refr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr rB � gslib.utils.encryption_helperr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr rO rb rh r r4 r4 r4 r5 �<module> s6 � �� __pycache__/rpo.cpython-39.pyc 0000644 00000013722 15025166704 0012220 0 ustar 00 a +(Wgc � @ s2 d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ dZd�d�e��Z d�e �Z dZ e e � d� Z dZ!dZ"de" e! Z#ee e#�Z$ee e!�Z%ee e"�Z&eg d�i d�Z'eg d�i d�Z(G dd � d e�Z)dS ) z/This module provides the rpo command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)Z ASYNC_TURBO�DEFAULTz({})�|z+ gsutil rpo set {} gs://<bucket_name>... z( gsutil rpo get gs://<bucket_name>... � am <B>SET</B> The ``rpo set`` command configures turbo replication for dual-region Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use turbo replication: gsutil rpo set ASYNC_TURBO gs://redbucket gs://bluebucket Configure your buckets to NOT use turbo replication: gsutil rpo set DEFAULT gs://redbucket gs://bluebucket z� <B>GET</B> The ``rpo get`` command returns the replication setting for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if your buckets are using turbo replication: gsutil rpo get gs://redbucket gs://bluebucket a The ``rpo`` command is used to retrieve or configure the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_ of dual-region Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. )�storage�buckets�listzL--format=value[separator=": "](format("gs://{}", name),rpo.yesno(no="None"))z--raw��gcloud_command�flag_map)r r �updatez--recovery-point-objectivec @ s� e Zd ZdZejdg ededdddej gej e � d�ge de e �d�e �� gd �d � Zejdg d d eeed �d �Zeeed �i d�Zdd� Zdd� Zdd� Zdd� Z dd� Z dS )� RpoCommandz"Implements the gsutil rpo command.�rpo� � F� �mode)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzConfigure replication)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r: �!/opt/gsutil/gslib/commands/rpo.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z7RpoCommand._ValidateBucketListingRefAndReturnBucketNamec C sF |j }| jj|jdg|jd�}|j}t|��d�}td||f � dS )z"Gets the rpo setting for a bucket.r ��fields�provider�/z%s: %sN) r5 � gsutil_api� GetBucket� bucket_namer6 r �str�rstrip�print)r8 r9 � bucket_url�bucket_metadatar �bucketr: r: r; �_GetRpo� s �zRpoCommand._GetRpoc C sh |j }|}|tvr td�t���tj|d�}| j�d|t |�� d�f � | j j |j |dg|jd� dS )z"Sets the rpo setting for a bucket.z.Invalid value for rpo set. Should be one of {})r zSetting rpo %s for %sr@ r r= r )r5 �VALID_RPO_VALUESr �format�VALID_RPO_VALUES_STRING�apitools_messages�Bucket�logger�inforD rE rA � PatchBucketrC r6 )r8 r9 Z rpo_valuerG Zformatted_rpo_valuerH r: r: r; �_SetRpo� s$ �� � �zRpoCommand._SetRpoc C s | j �d�}|dvr td��d}g }|dkr4| j}n$|dkrX| j}| j �d�}|�|� | j�d�tj kr�td� t � d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j�| jj�tj kr�td� t � d | j ����d }|| g|�R � q�q�|�sttt|� ��dS ) z-Handles rpo command on Cloud Storage buckets.r r zrpo only supports get|setNr r! r4 r z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS buckets.T)�args�popr rJ rS �appendrA �GetApiSelectorr �JSON�join�textwrap�wrapr7 �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr5 r6 r r ) r8 � subcommand�subcommand_func�subcommand_args� setting_arg� some_matched�url_args�url_str� bucket_iter�bucket_listing_refr: r: r; �_Rpo� sN �� ����zRpoCommand._Rpoc C sZ | j d }| jdd� |dks&|dkrJtj| jd� tj|gd� | �� S td| ��d S ) z(Command entry point for the rpo command.r T)� check_argsr r! )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rT � ParseSubOptsr �LogCommandParamsri rg r )r8 �action_subcommandr: r: r; � RunCommand� s �zRpoCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rX r �MakeNCloudURLsArgumentr rK �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr � GET_COMMAND� SET_COMMAND�gcloud_storage_mapr< rJ rS rg rn r: r: r: r; r c sP ����� �� .r )*rr � __future__r r rZ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rN �gslib.utils.constantsr �gslib.utils.shim_utilr rK rL rY rM � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprt �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONry r{ rz r} r~ r r: r: r: r; �<module> sL � �� �� __pycache__/perfdiag.cpython-39.pyc 0000644 00000174265 15025166704 0013213 0 ustar 00 a +(Wg�u � @ sB d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZdd lm Z dd lm Z dd lm Z dd lm Z ddl Z dd l m!Z! dd l m"Z" ddl#m$Z$ ddl#m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z4 ddl5m6Z6 ddl7m8Z8 ddl7m9Z9 ddl:m;Z; ddl<m=Z= dd l<m>Z> dd l?m@Z@ dd lAmBZB dd lAmCZC dd lAmDZD dd!lAmEZE dd"lAmFZF dd#lGmHZH dd$lGmIZI dd%lGmJZJ dd&lGmKZK dd'lGmLZL d(ZMd)eM d* ZNe d+d,�ZOe d-d.�ZPe d/d0�ZQe d1d2�ZRe d3d4�ZSi ZTG d5d6� d6eU�ZVG d7d8� d8eV�ZWdQd9d:�ZXdRd;d<�ZYdSd=d>�ZZdTd?d@�Z[dUdAdB�Z\dCdD� Z]dEdF� Z^G dGdH� dHe_�Z`dVdKdL�ZaG dMdN� dNe$�ZbdOdP� ZcdS )Wz%Contains the perfdiag gsutil command.� )�absolute_import)�print_function)�division)�unicode_literalsN)� defaultdict)� namedtuple)� cStringIO)� http_client)�range)�NotFoundException)�ServiceException)�Command)�DummyArgChecker)�CommandArgument)�config)� ApiSelector)�CommandException)�FilePart)�StorageUrlFromString)�storage_v1_messages)� text_util)�GetMaxRetryDelay)�ResumableThreshold)�UTF8)�GetCloudApiInstance)� GetDownloadSerializationData)�"CalculateB64EncodedMd5FromContents)�CheckFreeSpace)�GetDiskCounters)� GetFileSize)�IS_LINUX)�IsRunningInCiEnvironment)� DivideAndCeil)�HumanReadableToBytes)�MakeBitsHumanReadable)�MakeHumanReadable)� Percentilez� gsutil perfdiag [-i in.json] gsutil perfdiag [-o out.json] [-n objects] [-c processes] [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory] [-t tests] [-j ratio] gs://<bucket_name>... z <B>SYNOPSIS</B> a�! <B>DESCRIPTION</B> The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud Storage bucket. The ``bucket_name`` parameter must name an existing bucket to which the user has write permission. Several test files will be uploaded to and downloaded from this bucket. All test files will be deleted at the completion of the diagnostic if it finishes successfully. For a list of relevant permissions, see `Cloud IAM permissions for gsutil commands <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_. gsutil performance can be influenced by a number of factors originating at the client, server, or network level. Some examples include the following: + CPU speed + Available memory + The access path to the local disk + Network bandwidth + Contention and error rates along the path between gsutil and Google servers + Operating system buffering configuration + Firewalls and other network elements The `perfdiag` command is provided so that customers can run a known measurement suite when troubleshooting performance problems. <B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B> If the Cloud Storage team asks you to run a performance diagnostic please use the following command, and email the output file (output.json) to the @google.com address provided by the Cloud Storage team: gsutil perfdiag -o output.json gs://your-bucket Additional resources for discussing ``perfdiag`` results include the `Stack Overflow tag for Cloud Storage <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil/issues>`_. <B>OPTIONS</B> -n Sets the number of objects to use when downloading and uploading files during tests. Defaults to 5. -c Sets the number of `processes <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use while running throughput experiments. The default value is 1. -k Sets the number of `threads <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process to use while running throughput experiments. Each process will receive an equal number of threads. The default value is 1. NOTE: All specified threads and processes will be created, but may not by saturated with work if too few objects (specified with ``-n``) and too few components (specified with ``-y``) are specified. -p Sets the type of parallelism to be used (only applicable when threads or processes are specified and threads * processes > 1). The default is to use ``fan``. Must be one of the following: fan Use one thread per object. This is akin to using gsutil ``-m cp``, with sliced object download / parallel composite upload disabled. slice Use Y (specified with ``-y``) threads for each object, transferring one object at a time. This is akin to using parallel object download / parallel composite upload, without ``-m``. Sliced uploads not supported for s3. both Use Y (specified with ``-y``) threads for each object, transferring multiple objects at a time. This is akin to simultaneously using sliced object download / parallel composite upload and ``gsutil -m cp``. Parallel composite uploads not supported for s3. -y Sets the number of slices to divide each file/object into while transferring data. Only applicable with the slice (or both) parallelism type. The default is 4 slices. -s Sets the size (in bytes) for each of the N (set with ``-n``) objects used in the read and write throughput tests. The default is 1 MiB. This can also be specified using byte suffixes such as 500K or 1M. NOTE: these values are interpreted as multiples of 1024 (K=1024, M=1024*1024, etc.) NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with ``-n``) times as much disk space as specified will be required for the operation. -d Sets the directory to store temporary local files in. If not specified, a default temporary directory will be used. -t Sets the list of diagnostic tests to perform. The default is to run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a comma-separated list containing one or more of the following: lat For N (set with ``-n``) objects, write the object, retrieve its metadata, read the object, and finally delete the object. Record the latency of each operation. list Write N (set with ``-n``) objects to the bucket, record how long it takes for the eventually consistent listing call to return the N objects in its result, delete the N objects, then record how long it takes listing to stop returning the N objects. rthru Runs N (set with ``-n``) read operations, with at most C (set with -c) reads outstanding at any given time. rthru_file The same as ``rthru``, but simultaneously writes data to the disk, to gauge the performance impact of the local disk on downloads. wthru Runs N (set with ``-n``) write operations, with at most C (set with ``-c``) writes outstanding at any given time. wthru_file The same as wthru, but simultaneously reads data from the disk, to gauge the performance impact of the local disk on uploads. -m Adds metadata to the result JSON file. Multiple ``-m`` values can be specified. Example: gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname Each metadata key will be added to the top-level "metadata" dictionary in the output JSON file. -o Writes the results of the diagnostic to an output file. The output is a JSON file containing system information and performance diagnostic results. The file can be read and reported later using the ``-i`` option. -i Reads the JSON output file created using the ``-o`` command and prints a formatted description of the results. -j Applies gzip transport encoding and sets the target compression ratio for the generated test files. This ratio can be an integer between 0 and 100 (inclusive), with 0 generating a file with uniform data, and 100 generating random data. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. See `cp -j <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for specific semantics. <B>MEASURING AVAILABILITY</B> The ``perfdiag`` command ignores the boto num_retries configuration parameter. Instead, it always retries on HTTP errors in the 500 range and keeps track of how many 500 errors were encountered during the test. The availability measurement is reported at the end of the test. Note that HTTP responses are only recorded when the request was made in a single process. When using multiple processes or threads, read and write throughput measurements are performed in an external process, so the availability numbers reported won't include the throughput measurements. <B>NOTE</B> The ``perfdiag`` command runs a series of tests that collects system information, such as the following: + Retrieves requester's IP address. + Executes DNS queries to Google servers and collects the results. + Collects network statistics information from the output of ``netstat -s`` and evaluates the BIOS product name string. + If a proxy server is configured, attempts to connect to it to retrieve the location and storage class of the bucket being used for performance testing. None of this information will be sent to Google unless you proactively choose to send it. � FileDataTuplez size md5 data�FanDownloadTuplez6need_to_slice object_name file_name serialization_data�SliceDownloadTuplez<object_name file_name serialization_data start_byte end_byte�FanUploadTuplez9need_to_slice file_name object_name use_file gzip_encoded�SliceUploadTuplez@file_name object_name use_file file_start file_size gzip_encodedc @ s e Zd ZdZdS )�Errorz%Base exception class for this module.N��__name__� __module__� __qualname__�__doc__� r2 r2 �&/opt/gsutil/gslib/commands/perfdiag.pyr, 2 s r, c @ s e Zd ZdZdS )�InvalidArgumentz)Raised on invalid arguments to functions.Nr- r2 r2 r2 r3 r4 7 s r4 c C s@ t | |�}|jr&| �|j|j|j� n| �|j||j|j� dS )a Function argument to apply for performing fanned parallel downloads. Args: cls: The calling PerfDiagCommand class instance. args: A FanDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. N)r � need_to_slice�PerformSlicedDownload� object_name� file_name�serialization_data�Download��cls�args� thread_state� gsutil_apir2 r2 r3 �_DownloadObject< s ��r@ c C s, t | |�}| �|j||j|j|j|j� dS )a Function argument to apply for performing sliced downloads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. N)r r: r7 r8 r9 � start_byte�end_byter; r2 r2 r3 �_DownloadSliceM s �rC c C sN t | |�}|jr.| j|j|j|j||jd� n | j|j|j||j|jd� dS )a Function argument to apply for performing fanned parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A FanUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. �� gzip_encodedN)r r5 �PerformSlicedUploadr8 r7 �use_filerE �Uploadr; r2 r2 r3 � _UploadObjectZ s ��rI c C s2 t | |�}| j|j|j||j|j|j|jd� dS )a Function argument to apply for performing sliced parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. rD N)r rH r8 r7 rG � file_start� file_sizerE r; r2 r2 r3 � _UploadSliceq s �rL c C s t | |�}| �||� dS )a Function argument to apply for performing parallel object deletions. Args: cls: The calling PerfDiagCommand class instance. object_name: The object name to delete from the test bucket. thread_state: gsutil Cloud API instance to use for the operation. N)r �Delete)r< r7 r> r? r2 r2 r3 �_DeleteWrapper� s rN c C s | j �t|�� dS �z9Simple exception handler to allow post-completion status.N��logger�error�str)r< �er2 r2 r3 �_PerfdiagExceptionHandler� s rU c C s d S �Nr2 )�_r2 r2 r3 �_DummyTrackerCallback� s rX c @ s$ e Zd ZdZdZdd� Zdd� ZdS )� DummyFilezDA dummy, file-like object that throws away everything written to it.Zbwc O s d S rV r2 )�selfr= �kwargsr2 r2 r3 �write� s zDummyFile.writec C s d S rV r2 �rZ r2 r2 r3 �close� s zDummyFile.closeN)r. r/ r0 r1 �moder\ r^ r2 r2 r2 r3 rY � s rY �d � P c C sx |d }t �t||��}d}||k rtt||| �}t|d| �}|| }| �|d|� � | �d| � ||7 }q dS )am Writes data into a file like object. Args: fp: A file like object to write the data to. file_size: The amount of data to write to the file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. max_unique_random_bytes: The maximum number of bytes to generate pseudo-randomly before beginning to repeat bytes. The default was chosen as the next prime larger than 5 MiB. g Y@r � N� x)�os�urandom�min�intr\ ) �fprK � random_ratioZmax_unique_random_bytesZ random_bytesZtotal_bytes_written� num_bytesZ num_bytes_seqZnum_bytes_randomr2 r2 r3 �_GenerateFileData� s rk c @ s� e Zd ZdZejdg d�eddddddejej gej e � d�gd� Z ej dg d d ei d �Zd Zd ZdZdZdZdZdZdZdZdZeeeeeefZeeefZeeefZ e j � dde j j!j"j#�Z$e j �%ddd�Z&dZ'dZ(d Z)d Z*dZd d �Z+d!d"� Z,d[d%d&�Z-d'd(� Z.d)d*� Z/e0j1d+d,� �Z2d-d.� Z3d/d0� Z4G d1d2� d2e5j6�Z7d3d4� Z8d5d6� Z9d7d8� Z:d\d9d:�Z;d]d;d<�Z<d^d=d>�Z=d_d?d@�Z>dAdB� Z?d`dDdE�Z@dadFdG�ZAdHdI� ZBdJdK� ZCdLdM� ZDdNdO� ZEdPdQ� ZFdRdS� ZGdTdU� ZHdVdW� ZIdXdY� ZJdCS )b�PerfDiagCommandz*Implementation of gsutil perfdiag command.�perfdiag)Zdiag� diagnosticZperfZ performancer rb zn:c:k:p:y:s:d:t:m:i:o:j:F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRun performance diagnostic)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)r i i � � ZrthruZ rthru_fileZwthruZ wthru_file�lat�list�fan�slice�both� Credentials�gs_host�gs_port�P � � i @ g N@Tc C s� | j �d|� |rtjnd}tj|tj|d�}|�� \}}tjrTt|t �rT|� t �}|rv|j rvt d|j d�|�f ��|r~|S |j S )a� Executes a command in a subprocess. Args: cmd: List containing the command to execute. raise_on_error: Whether or not to raise an exception when a process exits with a non-zero return code. return_output: If set to True, the return value of the function is the stdout of the process. mute_stderr: If set to True, the stderr of the process is not printed to the console. Returns: The return code of the process or the stdout if return_output is set. Raises: Exception: If raise_on_error is set to True and any process exits with a non-zero return code. zRunning command: %sN)�stdout�stderrz8Received non-zero return code (%d) from subprocess '%s'.� )rQ �debug� subprocess�PIPE�Popen� communicate�six�PY3� isinstance�bytes�decoder � returncoder �join) rZ �cmd�raise_on_error� return_output� mute_stderrr� �pZ stdoutdatarW r2 r2 r3 �_Exec s �zPerfDiagCommand._Execc C s$ | j | j td�kr | j�d� dS )zBOutputs a warning message if a large amount of data is being used.�2GiBz2This is a large operation, and could take a while.N)� num_objects� thru_filesizer# rQ �infor] r2 r2 r3 �_WarnIfLargeData< s z PerfDiagCommand._WarnIfLargeData�gsutil_test_filer` c C s� t jd|| jdd�\}}t�|d�� }t|||� W d � n1 sF0 Y |sX|r�t|d��P}|rpt|�nd}|r�t|�nd} |r�|� � nd} t || | �t |< W d � n1 s�0 Y | j � |� |S )a� Creates a temporary file of the given size and returns its path. Args: file_size: The size of the temporary file to create. mem_metadata: If true, store md5 and file size in memory at temp_file_dict[fpath].md5, tempfile_data[fpath].file_size. mem_data: If true, store the file data in memory at temp_file_dict[fpath].data prefix: The prefix to use for the temporary file. Defaults to gsutil_test_file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. Returns: The file path of the created temporary file. z.binF)�suffix�prefix�dir�text�wbN�rb)�tempfile�mkstemp� directoryrd �fdopenrk �openr r �readr' �temp_file_dict�temporary_files�add) rZ rK � mem_metadata�mem_datar� ri �fd�fpathrh �md5�datar2 r2 r3 � _MakeTempFileA s � * . z PerfDiagCommand._MakeTempFilec C s� i | _ t� | _t� | _d| _d| _tt�| _d| _ d| _ | j | j v rrg | _ | jD ] }| j|ddd�}| j �|� qP| j �| j| j| j| jf��r�| jdddd�| _| j �| j| jf�r�| j| jdd| jd�| _tj� | j�| _ | j �| j| jf��r�g | _ g | _ t | j!�}|| j| j" k�r�| j#�$d| j"t%| j�f � | �&� t'| j"�D ]:}| j| jd| jd�}| j �|� | j �tj� |�� �qJn t(d t%| j| j" �t%|�f ��t)� | _*| j#�+| �,� � d S ) z?Performs setup operations needed before diagnostics can be run.r FT)r� r� i P )r� r� ri z) Creating %d local files each of size %s.)r� ri zbNot enough free disk space for throughput files: %s of disk space required, but only %s available.N)-�results�setr� �temporary_objects�total_requests�request_errorsr rg �error_responses_by_code�connection_breaks�teardown_completed�LAT� diag_tests� latency_files�test_lat_file_sizesr� �append� intersection�RTHRU�WTHRU� RTHRU_FILE� WTHRU_FILE�tcp_warmup_filer� �gzip_compression_ratio�mem_thru_file_namerd �path�basename�mem_thru_object_name�thru_file_names�thru_object_namesr r� r� rQ r� r% r� r r rY � discard_sink� addFilter�_PerfdiagFilter)rZ rK r� Zfree_disk_spacerW r8 r2 r2 r3 �_SetUpi sp ��� ��� ���zPerfDiagCommand._SetUpc C sr | j sht�� z,| jD ]}t�|� q| jr8t�| j� W n t yL Y n0 | j D ]}| � || j � qTd| _ dS )zDPerforms operations to clean things up after performing diagnostics.TN) r� r� �clearr� rd �remove�delete_directory�rmdirr� �OSErrorr� rM r? )rZ r� r7 r2 r2 r3 � _TearDown� s zPerfDiagCommand._TearDownc c sH | j �d|� t�� }dV t�� }|| �|| � | j �d|� dS )a� A context manager that measures time. A context manager that prints a status message before and after executing the inner command and times how long the inner command takes. Keeps track of the timing, aggregated by the given key. Args: key: The key to insert the timing value into a dictionary bucket. bucket: A dictionary to place the timing value in. Yields: For the context manager. z%s starting...Nz%s done.)rQ r� �timer� )rZ �key�bucket�t0�t1r2 r2 r3 �_Time� s zPerfDiagCommand._Timec C sD d}d}d}d}d}|�s@t t�� d| d t� �}z |� }| jd7 _d}W q t| j��y< } z�|d7 }|| jkr�| j�d� W Y d}~�q@t |t ��r|j dkr�| j |j d7 < | jd7 _| j d7 _ |d7 }t�|� n� || jk�r(| j�d � W Y d}~�q@n| jd7 _W Y d}~qd}~0 0 q|S ) z�Runs an operation with retry logic. Args: func: The function to run. Returns: True if the operation succeeds, False if aborted. Fr N� rb Tz,Reached maximum total retries. Not retrying.i� z3Reached maximum server error retries. Not retrying.)rf �randomr r� �tuple� exceptions�MAX_TOTAL_RETRIESrQ r� r� r �statusr� r� r� �sleep�MAX_SERVER_ERROR_RETRIESr� ) rZ �func�successZserver_error_retriedZ total_retried�iZ return_val� next_sleeprT r2 r2 r3 � _RunOperation� s> �&z PerfDiagCommand._RunOperationc s� t t��jd< t�j�D ]�}�j�d|d � �jD ]�}t| � �j � � �t j � |��_� j�t��}�j�d||�� t���� ���fdd�}��|� ���fdd�}��|�}t|������fd d �}��|� ���fd d �}��|� q4qd S )zRuns latency tests.�latencyz Running latency iteration %d...rb zN File of size %s located on disk at '%s' being diagnosed in the cloud at '%s'.c s` t �� j�} ��d� �jd ��* �jj| ���jdgd� W d � n1 sR0 Y d S )Nz UPLOAD_%dr� �name)�size�provider�fields)r� �BytesIOr� r� r� r? � UploadObjectr� )Zio_fp)� file_datarK rZ � upload_targetr2 r3 �_Upload; s �z1PerfDiagCommand._RunLatencyTests.<locals>._Uploadc sX �� d� �jd ��. �jj�j�j�jg d�d�W d � S 1 sJ0 Y d S )Nz METADATA_%dr� )r� � contentType� mediaLinkr� �r� r� )r� r� r? �GetObjectMetadata� bucket_namer7 r� r2 �rK rZ �urlr2 r3 � _MetadataF s �z3PerfDiagCommand._RunLatencyTests.<locals>._Metadatac sX �� d� �jd ��. �jj�j�j�j�j�d� W d � n1 sJ0 Y d S )Nz DOWNLOAD_%dr� )r� r9 )r� r� r? �GetObjectMediar r7 r� r� r2 )rK rZ r9 r r2 r3 � _DownloadR s �z3PerfDiagCommand._RunLatencyTests.<locals>._Downloadc sR �� d� �jd ��( �jj�j�j�jd� W d � n1 sD0 Y d S )Nz DELETE_%dr� �r� )r� r� r? � DeleteObjectr r7 r� r2 r r2 r3 �_Delete] s �z1PerfDiagCommand._RunLatencyTests.<locals>._DeleteN)r r� r� r r� rQ r� r� r� � bucket_url�Clonerd r� r� r7 r� r% � StorageUrlToUploadObjectMetadatar� r ) rZ r� r� Zreadable_file_sizer� r Zdownload_metadatar r r2 )r� rK rZ r9 r� r r3 �_RunLatencyTests' s. � z PerfDiagCommand._RunLatencyTestsc @ s e Zd Zdd� ZdS )z PerfDiagCommand._PerfdiagFilterc C s* |� � }d|v p&d|v p&d|v p&d|v S )NzCopying file:///z Copying gs://z Computing CRCzgsutil -m perfdiag)� getMessage)rZ �record�msgr2 r2 r3 �filterg s ��z&PerfDiagCommand._PerfdiagFilter.filterN)r. r/ r0 r r2 r2 r2 r3 r� e s r� c C s | j �t|�� dS rO rP )rZ rT r2 r2 r3 rU m s z)PerfDiagCommand._PerfdiagExceptionHandlerc C sh g }t t|��D ]0}|r || nd}|�t||| ||| �� q| jt|tdt| jj | j | j d� dS )ay Performs a parallel download of multiple objects using the fan strategy. Args: need_to_slice: If True, additionally apply the slice strategy to each object in object_names. object_names: A list of object names to be downloaded. Each object must already exist in the test bucket. file_names: A list, corresponding by index to object_names, of file names for downloaded data. If None, discard downloaded data. serialization_data: A list, corresponding by index to object_names, of serialization data for each object. N�r� r� �� arg_checker� parallel_operations_override� process_count� thread_count) r �lenr� r( �Applyr@ rU r �ParallelOverrideReason�PERFDIAG� processes�threads)rZ r5 � object_names� file_namesr9 r= r� r8 r2 r2 r3 �PerformFannedDownloadq s" ���z%PerfDiagCommand.PerformFannedDownloadc C s� |r:t |d�� }|�| j� W d � n1 s00 Y t| j| j�}g }t| j�D ]>}|| }t|d | d | jd �} |�t||||| �� qV| j t |t dt | j j| j| jd� dS )a3 Performs a download of an object using the slice strategy. Args: object_name: The name of the object to download. file_name: The name of the file to download data to, or None if data should be discarded. serialization_data: The serialization data for the object. �abNrb r r )r� �truncater� r" � num_slicesr rf r� r) r rC rU r r r r r ) rZ r7 r8 r9 rh �component_sizer= r� rA rB r2 r2 r3 r6 � s, * ���z%PerfDiagCommand.PerformSlicedDownloadc C sZ g }t t|��D ]"}|�t||| || ||�� q| jt|tdt| jj | j | j d� dS )aB Performs a parallel upload of multiple files using the fan strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data for file_name should be present in temp_file_dict if use_file is specified as False. Args: need_to_slice: If True, additionally apply the slice strategy to each file in file_names. file_names: A list of file names to be uploaded. object_names: A list, corresponding by by index to file_names, of object names to upload data to. use_file: If true, use disk I/O, otherwise read upload data from memory. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. r r N) r r r� r* r rI rU r r r r r )rZ r5 r! r rG rE r= r� r2 r2 r3 �PerformFannedUpload� s ���z#PerfDiagCommand.PerformFannedUploadc s0 t �j�j�}�fdd�t�j�D �}g }t�j�D ]<} | | } t|t| j| �}|�t||| || ||�� q4z��j t |t dt �j j�j�jd� g �t�j�D ] } tjj|| d�} ��| � q�� ���fdd�} ��| � W �j t|t dt �j j�j�jd� n$�j t|t dt �j j�j�jd� 0 dS ) a� Performs a parallel upload of a file using the slice strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data from for file_name should be present in temp_file_dict if use_file is specified as False. Args: file_name: The name of the file to upload. object_name: The name of the object to upload to. use_file: If true, use disk I/O, otherwise read upload data from memory. gsutil_api: CloudApi instance to use for operations in this thread. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a semaphore is used to limit resource usage. c s g | ]}� t |� �qS r2 )rS ��.0r� )r7 r2 r3 � <listcomp>� s z7PerfDiagCommand.PerformSlicedUpload.<locals>.<listcomp>r r )r� c s. t �� } �| _�jj| _� j�| �jd� d S �Nr )�apitools_messages�Objectr� r r r� � ComposeObjectr� )�dst_obj_metadata�r? r7 �request_componentsrZ r2 r3 �_Compose s �z5PerfDiagCommand.PerformSlicedUpload.<locals>._ComposeN)r" r� r% r rf r� r� r� r+ r rL rU r r r r r r, �ComposeRequest�SourceObjectsValueListEntryr� rN ) rZ r8 r7 rG r? rE r& Zcomponent_object_namesr= r� Zcomponent_start�src_obj_metadatar2 r2 r0 r3 rF � sf � � ��� �� � �z#PerfDiagCommand.PerformSlicedUploadc C s� |rdnd}|rdnd}| j �d|| jt| j�f � | �� | j| j| j| jd�| j |< |�r| j }| j }g }t | j�D ]�}| j �| j | � | j| jv r�| jj| jj| j | ddg| jjd �}n | �| j | | j | | j|�}t�| j | � t| j | d ��� |�t |�� qznF| j �| j � | �| j | j | j|�}d }| j g| j }t |�g| j }tj � | j!�} | j �| � | �| j!| | j� | �"| | j� t#�#� } | jd k�r�| jd k�r�t | j�D ]2}|�r�|| nd } | �"|| | j| || � �q�nz| j| j$| j%fv �r*| j| j%k} | �&| |||� nH| j| j'k�rrt | j�D ].}|�rT|| nd } | �(|| | || � �qBt#�#� } | | }| j| j }|| }|| j | d < || j | d< || j | d< d S )zRuns read throughput tests.�read_throughput_file�read_throughput� with file I/O� z9 Running read throughput tests %s (%s objects of size %s))rK r r � parallelismr� r �r� r� r# Nrb � time_took�total_bytes_copied�bytes_per_second))rQ r� r� r% r� r� r r �parallel_strategyr� r� r� r r� r� r� r� r? r r r �schemerH rd �unlinkr� r^ r� r r� r� r� r� r� r: r� �FAN�BOTHr" �SLICEr6 )rZ rG � test_name�file_io_stringr! r r9 r� � obj_metadata�warmup_obj_namer� r8 r5 r� r< r= r> r2 r2 r3 �_RunReadThruTests s� ���� � ��� � � � z!PerfDiagCommand._RunReadThruTestsc s� |rdnd}|rdnd}� j �d|� jt� j�f � � �� � j� j� j� j� j � j d�� j |< t j �� j�}� j�|� � �� j|� j� |r�� j}� j}n&� jg� j }� fdd�t� j�D �}|D ]}� j�|� q�t�� }� jd k�r0� jd k�r0t� j�D ]&} � j|| || � j|� j d � �qnx� j� j� jfv �rh� j� jk} � j| |||� j d � n@� j� j k�r�t� j�D ]&} � j || || |� j� j d � �q�t�� } | | } � j� j } | | }| � j | d < | � j | d < |� j | d < dS )z Runs write throughput tests.�write_throughput_file�write_throughputr8 r9 z: Running write throughput tests %s (%s objects of size %s))rK r r r: �gzip_encoded_writesr� c s g | ]}� j t|� �qS r2 )r� rS r( r] r2 r3 r* � s z6PerfDiagCommand._RunWriteThruTests.<locals>.<listcomp>rb rD r< r= r> N) rQ r� r� r% r� r� r r r? rL r� r� rd r� r� r� r� r� rH r? r� r� r� r r� rB rC r' rD rF )rZ rG rE rF rH r! r r7 r� r� r5 r� r< r= r> r2 r] r3 �_RunWriteThruTestss sz ���� � � � � z"PerfDiagCommand._RunWriteThruTestsc sT d�j i�jd< g }g }d�dd� td�D ��}d| d �t�j �D ]N}�jd d d �d �}tj�|�}|�|� |�t d ||d d �� �j � |� qF�j � d �j � �jt|ttd� g �g �t�� }t|��t� ����fdd�� � ���fdd�}�j � d�jj�j � �� �rT��|� �� �rt�� | �jk�r�j �d� �qT�qt�� } t����| | d��jd d< dd� |D �}�j � d�j � �jt|ttd� �j � d�jj�j � g �g �t�� }t|����r(� ��fdd�} ��| � ��r�t�� | �jk�rڈj �d� �q(�q�t�� } t����| | d��jd d < d S ) z0Runs eventual consistency listing latency tests.� num_files�listingr9 c S s g | ]}t �tj��qS r2 )r� �choice�string�ascii_lowercase)r) rW r2 r2 r3 r* � � z1PerfDiagCommand._RunListTests.<locals>.<listcomp>r� zgsutil-perfdiag-list-�-r T)r� r� r� Fz' Writing %s objects for listing test...)r c sp t � � } t�jj�jj�d�jdgd��}t|��jkrH�j � d�j� t � � }� � || � t dd� |D ��S )z>Lists and returns objects in the bucket. Also records latency.�/z items/name)r� � delimiterr� r� z5Listing produced more than the expected %d object(s).c S s g | ] }|j j�qS r2 )r� r� )r) �objr2 r2 r3 r* � rS z@PerfDiagCommand._RunListTests.<locals>._List.<locals>.<listcomp>) r� r� r? � ListObjectsr r r� r r� rQ �warningr� r� )r� �objectsr� )�list_latencies� list_prefixrZ r2 r3 �_List� s" ���z,PerfDiagCommand._RunListTests.<locals>._Listc s& � � } �� | �@ � ��t��� d S rV )�updater� r ��names)r] �expected_objects� files_seen� found_objectsr2 r3 �_ListAfterUpload� s z7PerfDiagCommand._RunListTests.<locals>._ListAfterUploadz5Listing bucket %s waiting for %s objects to appear...z)Maximum time reached waiting for listing.)�num_listing_callsr[ �files_seen_after_listingr< �insertc S s g | ]}|�qS r2 r2 )r) r7 r2 r2 r3 r* rS z'Deleting %s objects for listing test...z8Listing bucket %s waiting for %s objects to disappear...c s" � � } �� | � ��t��� d S rV )�intersection_updater� r r_ )r] rb rc r2 r3 �_ListAfterDelete s z7PerfDiagCommand._RunListTests.<locals>._ListAfterDelete�deleteN)r� r� r� r r� rd r� r� r� r* r� r� rQ r� r rI rU r r� r� r r r� �MAX_LISTING_WAIT_TIMErY r rN ) rZ Z list_objectsr= Z random_idrW r� r7 Ztotal_start_timerd Ztotal_end_timeri r2 )r] ra rb rc r[ r\ rZ r3 � _RunListTests� s� � � �� � ��� � �z PerfDiagCommand._RunListTestsNc s� d��du rt | j��j�� }||_t|��zz|rBt||���n t | j||� � } t� | ��������fdd�� � fdd�} �r�| n� } �� | �W �r��� � S n�r��� � 0 dS )a� Performs an upload to the test bucket. The file is uploaded to the bucket referred to by self.bucket_url, and has name object_name. Args: file_name: The path to the local file, and the key to its entry in temp_file_dict. object_name: The name of the remote object. gsutil_api: Cloud API instance to use for this upload. use_file: If true, use disk I/O, otherwise read everything from memory. file_start: The first byte in the file to upload to the object. (only should be specified for sliced uploads) file_size: The size of the file to upload. (only should be specified for sliced uploads) gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. Returns: Uploaded Object Metadata. Nc sH � t � k r&�j���j� g d��d�S �j���j� g d�t�d�S d S )N)r� r r� )r� r� r� rE )r� r� r� �tracker_callbackrE )r r� r� �UploadObjectResumablerX r2 )rK rh r? rE rZ r� r2 r3 � _InnerUploadX s" ��z,PerfDiagCommand.Upload.<locals>._InnerUploadc s2 t jj� � � W d � S 1 s$0 Y d S rV )�gslib�command�!concurrent_compressed_upload_lockr2 )ro r2 r3 �_InnerUploadLockm s z0PerfDiagCommand.Upload.<locals>._InnerUploadLock) r� r� r r r7 r r r� r� r� r� r^ ) rZ r8 r7 r? rG rJ rK rE � upload_urlr� rs Z upload_funcr2 )ro rK rh r? rE rZ r� r3 rH , s, � �zPerfDiagCommand.Uploadc sn d�zV|dur$t |d������ n�j�� ������fdd�}��|� W �rj��� n�rh��� 0 dS )a� Downloads an object from the test bucket. Args: object_name: The name of the object (in the test bucket) to download. gsutil_api: CloudApi instance to use for the download. file_name: Optional file name to write downloaded data to. If None, downloaded data is discarded immediately. serialization_data: Optional serialization data, used so that we don't have to get the metadata before downloading. start_byte: The first byte in the object to download. (only should be specified for sliced downloads) end_byte: The last byte in the object to download. (only should be specified for sliced downloads) Nzr+bc s" �j �jj���j�� �d� d S )N)r� rA rB r9 )r r r r� r2 �rB rh r? r7 rZ r9 rA r2 r3 �_InnerDownload� s �z0PerfDiagCommand.Download.<locals>._InnerDownload)r� �seekr� r� r^ )rZ r7 r? r8 r9 rA rB rv r2 ru r3 r: w s �zPerfDiagCommand.Downloadc s6 z � ��fdd�}�� |� W n ty0 Y n0 dS )z�Deletes an object from the test bucket. Args: object_name: The name of the object to delete. gsutil_api: Cloud API instance to use for this delete. c s � j �jj��jd� d S r+ )r r r r� r2 �r? r7 rZ r2 r3 � _InnerDelete� s �z,PerfDiagCommand.Delete.<locals>._InnerDeleteN)r� r )rZ r7 r? ry r2 rx r3 rM � s zPerfDiagCommand.Deletec C s. d}g }t dd��P}|�� dd� }|D ]*}|�� \}}}}|d �� r(|�|� q(W d � n1 sh0 Y i }t dd���}|D ]�}|�� dd� } | \ }}}} }} } } }}}||v r�t| �| } t|�| }t| �} t| �} t| �} t|�}| | | || |f||< q�W d � n1 �s 0 Y |S ) a� Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. i z/proc/partitions�rr� N���z/proc/diskstats� )r� � readlines�split�isdigitr� rg )rZ � sector_size� partitions�f�lines�linerW r� �retdict�values�reads�rbytes�rtime�writes�wbytes�wtimer2 r2 r3 �_GetDiskCounters� s, * 6z PerfDiagCommand._GetDiskCountersc C s� z| j ddgddd�}W n ty8 | j�d� Y dS 0 |�� �� }d}d}d}d}|�d�D ]�}d |v std |v rxd}|r�|du r�d |v s�d |v s�d |v r�d�dd� |D ��}|r�|du r�d|v s�d|v r�d�dd� |D ��}|r`|du r`d|v �sd|v �sd|v r`d�dd� |D ��}q`i }z(t|�|d< t|�|d< t|�|d< W n0 t t f�yz d|d< d|d< d|d< Y n0 |S )z�Tries to parse out TCP packet information from netstat output. Returns: A dictionary containing TCP information, or None if netstat is not available. �netstat�-sTF)r� r� zGnetstat not found on your system; some measurement data will be missingN� ztcp:ztcp statisticszsegments retransmitedzretransmit timeoutszsegments retransmittedr9 c s s | ]}|t jv r|V qd S rV �rQ �digits�r) �cr2 r2 r3 � <genexpr>� rS z/PerfDiagCommand._GetTcpStats.<locals>.<genexpr>zsegments receivedzpackets receivedc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� rS zsegments send outz packets sentz segments sentc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� rS �tcp_retransmit� tcp_received�tcp_sent) r� r� rQ rY �strip�lowerr~ r� rg � ValueError� TypeError)rZ �netstat_outputZ found_tcpr� r� r� r� �resultr2 r2 r3 � _GetTcpStats� s` � ��� �� ��� z PerfDiagCommand._GetTcpStatsc C s� i }t jt jt jt jf}tj�ddd�|d< tj�ddd�}tj�ddd�}t |�|d < tj�dd |rjdnd �r|| j � d � zt � t � � �|d < W n |y� d|d < Y n0 | j|d< t�dt�� �|d< dd| jg}z8| j|ddd�}t�d|�}|�r|�d�nd|d< W n ttf�y0 d|d< Y n0 t�r�z@tdd�� }|�� dk|d< W d � n1 �sl0 Y W n t�y� Y n0 |�dd ��r�t � � } d d d d d | g}z t � } | j|d| d�|d!< W n ttf�y� d|d!< Y n0 | j j | j j d"d#g| j j!d$�} | j"|d%< | j#|d&< z,t�� } t � | j� t�� } | | |d'< W n |�yf Y n0 z t �$| j�\} }}||d(< W n |�y� g }g |d(< Y n0 g |d)< |D ]>}z"t �%|�\} }}|d) �&| � W n |�y� Y n0 �q�z@g d*�}| j|ddd�}t�d+|�}|�r*|�d,�nd|d-< W n ttf�yT d|d-< Y n0 i |d.< |D ]X}z<t � � }t�� } |�'|| j(f� t�� } | | |d. |< W n |�y� Y n0 �qb|�rXd}z*t�� } t � |�}t�� } | | |d/< W n |�y Y n0 z<t � � }t�� } |�'|�p$||f� t�� } | | |d0< W n |�yV Y n0 zt)�*� |d1< W n t+�y� d|d1< Y n0 zt,t-�.� �|d2< W n t/tf�y� d|d2< Y n0 d}d}d}d}z�td3d���}|D ]�}|�0d4��r t1d�2d5d6� |D ���d7 }n||�0d8��r6t1d�2d9d6� |D ���d7 }nR|�0d:��r`t1d�2d;d6� |D ���d7 }n(|�0d<��r�t1d�2d=d6� |D ���d7 }�q�W d � n1 �s�0 Y W n t3t4f�y� Y n0 ||||d>�|d?< i |d@< t5t�D ]>}t6t|�}|�7� �r�t8|t9j:��rdA|v �s�||d@ |< �q�i |dB< g dC�} | D ]l} zPt| d��}|�;� } W d � n1 �sn0 Y | �<� |dB t-j=�>| �< W n t3�y� Y n0 �q>|| j?dD< dS )Ez Collects system information.�Boto� is_secureT�boto_https_enabled�proxyN� proxy_portr � using_proxy� proxy_rdnsFz�DNS lookups are disallowed in this environment, so some information is not included in this perfdiag run. To allow local DNS lookups while using a proxy, set proxy_rdns to False in your boto file.� ip_addressr9 �tempdir�%a, %d %b %Y %H:%M:%S +0000� gmt_timestamp�nslookupz -type=CNAME)r� r� z = (?P<googserv>[^.]+)\.Zgoogserv�googserv_routez /sys/class/dmi/id/product_namerz zGoogle Compute Engine �on_gce�gcloudZcomputeZ instancesr� z --filter=�gce_instance_info�location� storageClassr; �bucket_location�bucket_storageClass�google_host_dns_latency� googserv_ips�googserv_hostnames)r� z -type=TXTzo-o.myaddr.google.com.z text\s+=\s+"(?P<dnsip>[\.\d]+)"Zdnsip� dns_o-o_ip� google_host_connect_latencies�proxy_dns_latency�proxy_host_connect_latency� cpu_count�load_avgz /proc/meminfoZMemTotalc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS z2PerfDiagCommand._CollectSysInfo.<locals>.<genexpr>�� ZMemFreec s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS ZBuffersc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS ZCachedc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS )� mem_total�mem_free� mem_buffers� mem_cached�meminfoZ gsutil_configr� �tcp_proc_values)z /proc/sys/net/core/rmem_defaultz/proc/sys/net/core/rmem_maxz /proc/sys/net/core/wmem_defaultz/proc/sys/net/core/wmem_maxz!/proc/sys/net/ipv4/tcp_timestampsz/proc/sys/net/ipv4/tcp_sackz%/proc/sys/net/ipv4/tcp_window_scaling�sysinfo)@�socketrR �herror�gaierror�timeout�botor �get�getint�boolrQ r� � gethostbyname� gethostnamer� r� �strftime�gmtime� XML_API_HOSTr� �re�search�groupr r� r r� �readliner! r? � GetBucketr r r@ r� r� �gethostbyname_ex� gethostbyaddrr� �connect� XML_API_PORT�multiprocessingr� �NotImplementedErrorr� rd � getloadavg�AttributeError� startswithrg r� �IOErrorr� r� �getattr�isupperr� r� � string_typesr� r� r� r� r� ) rZ r� Z socket_errors� proxy_hostr� r� Znslookup_cname_output�mrh �hostnamer� Z bucket_infor� r� rW Z ipaddrlistZ googserv_ipZnslookup_txt_output�sock�proxy_ipr� r� r� r� r� r� �attr� attr_valueZstats_to_check�fname�valuer2 r2 r3 �_CollectSysInfo sJ � � � � 4� � � � � � �,� � ( z PerfDiagCommand._CollectSysInfoc s� t |�}tt|��| � t�t� fdd�|D ��| �}tjt|��d�ddd� tjd� d �d �ddd� tjd|d �d �ddd� tjdt |d �d �d �ddd� t�dt |d �d �d �d� dS )zAPrints out mean, standard deviation, median, and 90th percentile.c 3 s | ]}|� d V qdS )r� Nr2 )r) �x��meanr2 r3 r� � rS z0PerfDiagCommand._DisplayStats.<locals>.<genexpr>� r9 r� ��endz%.1fr� � � g �?r| g�������?N) r �float�sum�math�sqrtr � print_to_fdrS �rjustr& )rZ �trials�n�stdevr2 r� r3 � _DisplayStats� s ��z PerfDiagCommand._DisplayStatsc 1 C s t �� t �d� t �d�d�� t �d� d| jv �r�t �� t �d� t �d�d�� t �d� t �d� t �d� t| jd �D �]8}t| jd | �}|�d �\}}t|�}|d kr�t jd �d �d dd� t jt|��d �d dd� | � |� |dk�r:t jd�d �d dd� t jt|��d �d dd� | � |� |dk�r~t jd�d �d dd� t jt|��d �d dd� | � |� |dkr�t jd�d �d dd� t jt|��d �d dd� | � |� q�d| jv �r`t �� t �d� t �d�d�� t �d� | jd }t �d| j t|d �t|d �f � t �dt |d d � � d |v �r`t �d |d � d | jv �r�t �� t �d� t �d!�d�� t �d� | jd }t �d| j t|d �t|d �f � t �dt |d d � � d |v �r�t �d |d � d"| jv �r�t �� t �d� t �d#�d�� t �d� | jd" }t �d| j t|d �t|d �f � t �d$t |d d � � d |v �r�t �d |d � d%| jv �r:t �� t �d� t �d&�d�� t �d� | jd% }t �d| j t|d �t|d �f � t �d$t |d d � � d |v �r:t �d |d � d'| jv �rxt �� t �d� t �d(�d�� t �d� | jd' } | d) } | d* } t �d+| d, � t �d-| d. � t �d/| d0 � t �d1d2� d3d4� | d5 D �� � t �d6d2� t t| d7 �� � t �d8| d, � t �d-| d. � t �d/| d0 � t �d1d2� d9d4� | d5 D �� � t �d6d2� t t| d7 �� � d:| jv � r�t �� t �d� t �d;�d�� t �d� | jd: } t �d<| d= � t �d>| d? � t �d@| jdA � t �dB| j�dCdD� � t �dE| j�dFdD� � dG| v �r�| dG } dH}zt�| dI�}W n t�y\ Y n0 |�r�t�|�}tj�|�}t �dJ|�dK� � dL| v �r�t �dM| dL � | dL �r�t �dN| dO �dPdQ� � t �dR| dS � t �dT| dU � t �dV| dW � t �dXdY� | dZ � � t �d[dY� | d\ � � t �d]| d^ � t �d_| d` � t �da| db � zPt �dct| dd de � � t �dft| dd dg | dd dh | dd di � � W n t�y� Y n0 dj| v �rFdk| v �rF| dj }| dk }dlD ]J}z.|dm| |dm| }t �dn||f � W n t�y> Y n0 �q�n t �do� dp| v � rjdq| v � rjt jdrdd� | dp }| dq }t jd ds�dt�dd� duD ]}t j|�d �dd� �q�t �� t|�D ]�}|| }|| }|\}} } } } } |\}!}"}#}$}%}&t jd |�dt�dd� |!| |"| |#| |$| |%| |&| g}'|'D ] }t jt|��d �dd� � q>t �� �q�dv| v � r�t jdwdd� t�| dv �D ]}(t �dx|( � � q�dy| v � r�t �dz| dy � d{| v � r�t �d|| d{ � d}| v � r�t �d~| d} d � d�| v � rBt �d�� t�| d� �D ] \})}*t �d�|)|*d f � � q d�| v � rbt �d�| d� d � d�| v � r�t �d�| d� d � d�| jv � r�d�| jv � r�t �� t �d� t �d��d�� t �d� t| jd� �}+t| jd� �},t| jd� �}-|+d�k� r|+|, t |+� d� nd�}.t �d�|+ � t �d�|, � t �d�|- � t �d�|. � d�| jv � r�tt�| jd� ��}/|/� r�t �d�� t �dP� d�d4� |/D ��� | j � r�t | j d���"}0t j | j|0d�d�� W dH � n1 � s�0 Y t �� t �d�| j � t �� dHS )�z/Displays results collected from diagnostic run.zN==============================================================================zDIAGNOSTIC RESULTS�N r� zN------------------------------------------------------------------------------ZLatencyzOOperation Size Trials Mean (ms) Std Dev (ms) Median (ms) 90th % (ms)zO========= ========= ====== ========= ============ =========== ===========rW ZMETADATA�Metadatar� r9 r� r� �DOWNLOADr: �UPLOADrH �DELETErM rK zWrite Throughputz5Copied %s %s file(s) for a total transfer size of %s.rK r= zWrite throughput: %s/s.r> � r: zParallelism strategy: %srJ z Write Throughput With File I/Or7 zRead ThroughputzRead throughput: %s/s.r6 z Read Throughput With File I/OrO ZListingrg rj zAfter inserting %s objects:rN z0 Total time for objects to appear: %.2g secondsr< z" Number of listing calls made: %sre z) Individual listing call latencies: [%s]�, c s s | ]}d | V qdS �z%.2gsNr2 �r) r� r2 r2 r3 r� � rS z2PerfDiagCommand._DisplayResults.<locals>.<genexpr>r[ z' Files reflected after each call: [%s]rf zAfter deleting %s objects:c s s | ]}d | V qdS r r2 r r2 r2 r3 r� � rS r� zSystem InformationzIP Address: %sr� zTemporary Directory: %sr� zBucket URI: %s� bucket_urizgsutil Version: %s�gsutil_version�Unknownzboto Version: %s� boto_versionr� Nr� zMeasurement time: %sz%Y-%m-%d %I:%M:%S %p %Zr� zRunning on GCE: %szGCE Instance: %sr� r� z zBucket location: %sr� zBucket storage class: %sr� zGoogle Server: %sr� z!Google Server IP Addresses: %sz r� z Google Server Hostnames: %sr� z#Google DNS thinks your IP is: %sr� zCPU Count: %sr� zCPU Load Average: %sr� zTotal Memory: %sr� r� zFree Memory: %sr� r� r� � netstat_end� netstat_start)�sent�receivedZ retransmitztcp_%sz!TCP segments %s during test: %dzQTCP segment counts not available because "netstat" was not found during test runs�disk_counters_end�disk_counters_startzDisk Counter Deltas: Zdiskr� )r� r� r� r� r� r� r� zTCP /proc values: z %s = %sr� zBoto HTTPS Enabled: %sr� z$Requests routed through proxy: %sr� zALatency of the DNS lookup for Google Storage server (ms): %.1fg @�@r� z7Latencies connecting to Google Storage server IPs (ms):z %s = %.1fr� z@Latency of the DNS lookup for the configured proxy (ms): %.1fr� z8Latency connecting to the configured proxy (ms): %.1fr� r� zIn-Process HTTP Statisticsr� r r` z Total HTTP requests made: %dzHTTP 5xx errors: %dzHTTP connections broken: %dzAvailability: %.7g%%r� zError responses by code:c s s | ]}d | V qdS )z %s: %sNr2 r� r2 r2 r3 r� 2 rS �wr� )�indentz Output file written to '%s'.)!r r� �centerr� �sortedr~ rg r� r% r� r� r$ r� �maprS r� r� �strptimer� �calendar�timegm�datetime� fromtimestampr� �replacer� r� � iteritemsr� � output_filer� �json�dump)1rZ r� r� �op�numbytesZ write_thruZwrite_thru_fileZ read_thruZread_thru_filerO rg rj r� Z ts_string� timetuple� localtimeZlocaldtZ netstat_afterZnetstat_beforeZtcp_type�deltaZ disk_afterZ disk_beforeZcolnameZdiskname�before�afterZreads1Zwrites1Zrbytes1Zwbytes1Zrtime1Zwtime1Zreads2Zwrites2Zrbytes2Zwbytes2Zrtime2Zwtime2Zdeltas�item�ipr� �totalZ numerrorsZ numbreaksZ availabilityZ sorted_codesr� r2 r2 r3 �_DisplayResults s� ��� � � � ����� � ����� � ����� � ����� � ��������������� � � � �� � ��� � �� �� � � � � �� � �� �� ��� � 2z PerfDiagCommand._DisplayResultsc C s> z t |�}|dk rt|��|W S ty8 t|��Y n0 dS )a^ Tries to convert val argument to a positive integer. Args: val: The value (as a string) to convert to a positive integer. msg: The error message to place in the CommandException on an error. Returns: A valid positive integer. Raises: CommandException: If the supplied value is not a valid positive integer. rb N)rg r r� )rZ �valr r2 r2 r3 �_ParsePositiveInteger< s z%PerfDiagCommand._ParsePositiveIntegerc C sh d| _ d| _d| _d| _d| _d| _t�� | _d| _ t | j �| _ d| _ d| _i | _d| _d| _| j�r| jD �]�\}}|dkr�| �|d �| _ |d kr�| �|d �| _|d kr�| �|d �| _|dkr�|�� | jv r�|�� | _n td| ��|dkr�| �|d�| _|dk�r2zt|�| _W n t�y0 td��Y n0 |dk�rd|| _tj�| j��sdd| _ t� | j� |dk�r�t � | _ |� � � d�D ].}|�� | j v�r�td| ��| j � |� �q�|dk�r�|� d�}t!|�dk�r�td | ��|\}}|| j|< |d k�r tj�"|�| _ |d k�r�tj�"|�| _tj�#| j��s@td | ��zNt$| jd ��,}t%�&|�| _'| j(�)d!| j� W d � n1 �s�0 Y W n t�y� td"| ��Y n0 dS |d#krld| _zt*|�| _W n t�y� d$| _Y n0 | jd%k �s| jdkrltd&��ql| jdk�s&| jdk�r8| j�s8| j+| _n(| jdk�r`| jdk�r`| j�r`td'��| j,�sp| �-� t.| j,d% �| _/| j/j0| _1| j/�2� �s�| j/�3� �r�td(| j,d% ��| jtd)�k�r�| j4| j v �s�| j5| j v �r�td*��| j| j6| j7fv }| j1d+k�o| j �8| j5| j9�} |�r,| �r,td,��| j:j;| j/j<| j/j0d-gd.� t=j>t?j@t?jAt?jBt=jCtDg| _EdS )/z&Parses arguments for perfdiag command.r� rb N� r� Fr` z-nz,The -n parameter must be a positive integer.z-cz,The -c parameter must be a positive integer.z-kz,The -k parameter must be a positive integer.z-pz)'%s' is not a valid parallelism strategy.z-yz,The -y parameter must be a positive integer.r� zInvalid -s parameter.z-dTz-t�,z8List of test names (-t) contains invalid test name '%s'.z-m�:r� z,Invalid metadata key-value combination '%s'.z-oz-iz Invalid input file (-i): '%s'.rz zRead input file: '%s'.z'Could not decode input file (-i): '%s'.z-jr{ r z7The -j parameter must be between 0 and 100 (inclusive).zrCannot specify parallelism strategy (-p) without also specifying multiple threads and/or processes (-c and/or -k).zOThe perfdiag command requires a URL that specifies a bucket. "%s" is not valid.r� z~For in-memory tests maximum file size is 2GiB. For larger file sizes, specify rthru_file and/or wthru_file with the -t option.�s3zQSliced uploads are not available for s3. Use -p fan or sequential uploads for s3.�idr )Fr� r r r? r% r� r� � gettempdirr� r� r� �DEFAULT_DIAG_TESTSr� r � input_file� metadata_keysrL r� �sub_optsr. r� �PARALLEL_STRATEGIESr r# r� rd r� �exists�makedirsr� r~ �ALL_DIAG_TESTSr� r �abspath�isfiler� r �loadr� rQ r� rg rB r= �$RaiseWrongNumberOfArgumentsExceptionr r r@ r� � IsCloudUrl�IsBucketr� r� rD rC r� r� r? r� r r � HTTPExceptionr� rR r� r� � BadStatusLiner r� ) rZ �o�arE �piecesr� r� r� Z perform_sliceZslice_not_availabler2 r2 r3 � _ParseArgsQ s ��� �� � � 4� �� � � � �� �� � �zPerfDiagCommand._ParseArgsc C s� | � � | jr| �� dS tj�ddd� | j�d| j| j | j | j | j t | j�d�| j�| j| j� �z�| �� | �� | �� }|r�|| jd d< tr�t� | jd d < t| j �| jd < d | jd < | j| jd < | j| jv r�| � � | j | jv r�| � � | j | jv �r| j dd� | j!| jv �r(| j dd� | j"| jv �r>| � � | j#| jv �rT| �$� | �� }|�rp|| jd d< t�r�t� | jd d< | j%| jd< | j&| jd< | j'| jd< | j(| jd< t)j*| jd< tj+| jd< | �,� | �� W | �,� n | �,� 0 dS )z3Called by gsutil when the command is being invoked.r r� � num_retries�0z�Number of iterations to run: %d Base bucket URI: %s Number of processes: %d Number of threads: %d Parallelism strategy: %s Throughput file size: %s Diagnostics to run: %s Gzip compression ratio: %s Gzip transport encoding writes: %sr r� r r r rm Z json_format�metadataT)rG r r r� r� r� r� r r )-rH r6 r, r� r r� rQ r� r� r r r r? r% r� r� r� r� rL r� r� r� r� r r rS r7 r� r r� rI r� rM r� r� �LISTrl r� r� r� r� rp �VERSION� __version__r� )rZ r� r2 r2 r3 � RunCommand� sd � zPerfDiagCommand.RunCommand)TFF)r FFr� r` )F)F)F)F)Fr NF)NNr N)Kr. r/ r0 r1 r �CreateCommandSpec� _SYNOPSISr �XML�JSONr � MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� r� rL rB rD rC r< r5 r9 r� r r� �gs� connection� GSConnection� DefaultHostr� r� r� r� r� ZKEY_BUFFER_SIZErk r� r� r� r� r� � contextlib�contextmanagerr� r� r �logging�Filterr� rU r" r6 r' rF rI rM rl rH r: rM r� r� r� r� r, r. rH rO r2 r2 r2 r3 rl � s� �� � � # � (P* .> " � * � K X Hu � N � +)9 ` : rl c C s< | � � r,| �� r,t�� }| j|_| j|_|S td| ��d S )NzENon-cloud URL upload target %s was created in perfdiag implemenation.) rA �IsObjectr, r- r7 r� r r� r )� storage_urlr� r2 r2 r3 r 0 s �r )N)N)N)N)N)r r` ra )dr1 � __future__r r r r r � collectionsr r r] r r r_ r� r� rd r� r� r� rQ r� r� r� r� �boto.gs.connectionr� � six.movesr r r rp �gslib.cloud_apir r � gslib.commandr r �gslib.command_argumentr �gslib.commandsr �gslib.cs_api_mapr �gslib.exceptionr �gslib.file_partr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r, Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr � gslib.utils.cloud_api_helperr r �gslib.utils.hashing_helperr Zgslib.utils.system_utilr r r r r! �gslib.utils.unit_utilr" r# r$ r% r& rQ rW r' r( r) r* r+ r� � Exceptionr, r4 r@ rC rI rL rN rU rX �objectrY rk rl r r2 r2 r2 r3 �<module> s� �� < ���� � $ t __pycache__/du.cpython-39.pyc 0000644 00000020735 15025166704 0012032 0 ustar 00 a +(Wg�) � @ s0 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" ddlm#Z# dZ$de$ d Z%G dd� de �Z&dS ) zCImplementation of Unix-like du command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketListingObject)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)� ls_helper)�NO_MAX)�S3_DELETE_MARKER_GUID)�UTF8)�GcloudStorageFlag)�GcloudStorageMap)� print_to_fd)�MakeHumanReadable)� text_utilz gsutil du url... z <B>SYNOPSIS</B> ak <B>DESCRIPTION</B> The du command displays the amount of space in bytes used up by the objects in a bucket, subdirectory, or project. The syntax emulates the Linux ``du -b`` command, which reports the disk usage of files and subdirectories. For example, the following command reports the total space used by all objects and subdirectories under gs://your-bucket/dir: gsutil du -s -a gs://your-bucket/dir <B>OPTIONS</B> -0 Ends each output line with a 0 byte rather than a newline. You can use this to make the output machine-readable. -a Includes both live and noncurrent object versions. Also prints the generation and metageneration number for each listed object. If this flag is not specified, only live object versions are included. -c Includes a total size at the end of the output. -e Exclude a pattern from the report. Example: -e "*.o" excludes any object that ends in ".o". Can be specified multiple times. -h Prints object sizes in human-readable format. For example, ``1 KiB``, ``234 MiB``, or ``2GiB``. -s Displays only the total size for each argument, omitting the list of individual objects. -X Similar to ``-e``, but excludes patterns from the given file. The patterns to exclude should be listed one per line. <B>EXAMPLES</B> To list the size of each object in a bucket: gsutil du gs://bucketname To list the size of each object in the ``prefix`` subdirectory: gsutil du gs://bucketname/prefix/* To include the total number of bytes in human-readable form: gsutil du -ch gs://bucketname To see only the summary of the total number of (live) bytes in two given buckets: gsutil du -s gs://bucket1 gs://bucket2 To list the size of each object in a bucket with `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ enabled, including noncurrent objects: gsutil du -a gs://bucketname To list the size of each object in a bucket, except objects that end in ".bak", with each object printed ending in a null byte: gsutil du -e "*.bak" -0 gs://bucketname To list the size of each bucket in a project and the total size of the project: gsutil -o GSUtil:default_project_id=project-name du -shc c @ s� e Zd ZdZejdg ededdddej ej gej e � � gd� Z ejdg dd ei d �Zed dged �ed �ed�ed�ed�ed�ed�d�d�Zdd� Zdd� Zdd� ZdS ) � DuCommandz$Implementation of gsutil du command.Zdur z 0ace:hsX:FT) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzDisplay object size usage)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storagez--zero-terminatorz--all-versionsz--totalz--exclude-name-patternz--readable-sizesz --summarizez--exclude-name-pattern-file)�-0�-a�-c�-e�-h�-s�-X)�gcloud_command�flag_mapc C s< | j rt|�nt�|�}tjdj|t�|�d�| jd� d S )Nz{size:<11} {name})�size�name)�end) �human_readabler �six� text_typer r �format� ensure_text� line_ending)�self� num_bytesr4 � size_string� r? � /opt/gsutil/gslib/commands/du.py�_PrintSummaryLine� s � � ��zDuCommand._PrintSummaryLinec C s� |j }|j}|jr4t|jjv r4d}d}d}|d7 }n$| jrDt|j�nt|j�}|j}d}| j s�dj |t � |�t � | j �d�}t|tjdd� ||fS ) z�Print listing info for given bucket_listing_ref. Args: bucket_listing_ref: BucketListing being listed. Returns: Tuple (number of objects, object size) Raises: Exception: if calling bug encountered. �0r z<DeleteMarker>� z{size:<11} {url}{ending})r3 �urlZending� )�filer5 )� root_object� url_string�metadatar �additionalPropertiesr6 r r3 �str� summary_onlyr9 r7 r: r; r �sys�stdout)r< �bucket_listing_ref�obj�url_strr> r= Znum_objsZ url_detailr? r? r@ � _PrintInfoAboutBucketListingRef� s. � � � �z)DuCommand._PrintInfoAboutBucketListingRefc s� d� _ d� _d� _d� _d� _g � _� jr�� jD ]�\}}|dkrHd� _ q0|dkrXd� _q0|dkrhd� _q0|dkr~� j�|� q0|d kr�d� _q0|d kr�d� _q0|d kr0|d kr�tj }d}n"t j r�t |d �n t |d t d�}d}dd� |D �� _|r0|�� q0� j�s dg� _d}d}� fdd�}d&dd�}� fdd�} � jD �]H} t| �} | �� �r`td� j ��dg} tj� j� j|||| |� jd� j| d � } | �� �r�� �d | j �jd gd �}n6| �� �r� �d | j| j f �jd gd �}n t | �g}|D ]�}|j }|�� �r� j�rt|j d!d"��}| � |�\}}}|�!� �rT|dk�rTt"| ��rT� j�sTd}||7 }� j�r� �#||j$�%d#��&t �� �q�q:� j�r�� �#|d$� |�r�td%��dS )'z'Command entry point for the du command.� Fr* � r+ Tr, r- r. r/ r0 �-�r)�encodingc S s g | ]}t �|�� ��qS r? )r7 r: �strip)�.0�liner? r? r@ � <listcomp>� � z(DuCommand.RunCommand.<locals>.<listcomp>zgs://r c s � � | �S �N)rR )�blr�r< r? r@ �_PrintObjectLong� s z.DuCommand.RunCommand.<locals>._PrintObjectLongNc S s d S r] r? )Z unused_blrr? r? r@ � _PrintNothing� s z+DuCommand.RunCommand.<locals>._PrintNothingc s � j s� �| |j�t�� d S r] )rL rA rH �encoder )r= r^ r_ r? r@ �_PrintDirectory� s z-DuCommand.RunCommand.<locals>._PrintDirectoryz$Only cloud URLs are supported for %sr3 ) Zprint_object_funcZprint_dir_funcZprint_dir_header_funcZprint_dir_summary_funcZprint_newline_func� all_versionsZshould_recurse�exclude_patterns�fieldsz%s://*�id)� bucket_fieldsz%s://%sz**)�wildcard_suffix�/�totalz$One or more URLs matched no objects.)N)'r; rd Z produce_totalr6 rL re �sub_opts�appendrM �stdinr7 �PY2�openr �close�argsr � IsFileUrlr � command_namer ZLsHelper�WildcardIterator�logger� IsProvider�scheme� IterBuckets�IsBucket� bucket_namer � storage_url�CreatePrefixUrlZExpandUrlAndPrint�IsObjectr rA rH �rstriprb )r< �o�a�fZf_close� total_bytesZgot_nomatch_errorsr` ra rc �url_argZtop_level_storage_url�bucket_listing_fieldsZlisting_helperZtop_level_iterr^ r| �_Zexp_objsZ exp_bytesr? r_ r@ � RunCommand� s� �� ��� ��� ���� zDuCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_maprA rR r� r? r? r? r@ r z sL ��� �� #r )'r� � __future__r r r r �localerM r7 �gslib.bucket_listing_refr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r Z gslib.utilsr �gslib.utils.constantsr r r �gslib.utils.shim_utilr r Zgslib.utils.text_utilr �gslib.utils.unit_utilr r r� r� r r? r? r? r@ �<module> s< ��K __pycache__/mv.cpython-39.pyc 0000644 00000012076 15025166704 0012043 0 ustar 00 a +(Wgm � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd l m Z dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZde d ZG dd� de�ZdS )zCImplementation of Unix-like mv command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literals)�Command)�CommandArgument)�CP_AND_MV_SHIM_FLAG_MAP)� CP_SUB_ARGS)�'ShimTranslatePredefinedAclSubOptForCopy)� ApiSelector)�CommandException)�StorageUrlFromString)�NO_MAX)�GcloudStorageMapzb gsutil mv [-p] src_url dst_url gsutil mv [-p] src_url... dst_url gsutil mv [-p] -I dst_url z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The ``gsutil mv`` command allows you to move data between your local file system and the cloud, move data within the cloud, and move data between cloud storage providers. For example, to move all objects from a bucket to a local directory you could use: gsutil mv gs://my_bucket/* dir Similarly, to move all objects from a local directory to a bucket you could use: gsutil mv ./dir gs://my_bucket <B>RENAMING GROUPS OF OBJECTS</B> You can use the ``gsutil mv`` command to rename all objects with a given prefix to have a new prefix. For example, the following command renames all objects under gs://my_bucket/oldprefix to be under gs://my_bucket/newprefix, otherwise preserving the naming structure: gsutil mv gs://my_bucket/oldprefix gs://my_bucket/newprefix If you do a rename as specified above and you want to preserve ACLs, you should use the ``-p`` option (see OPTIONS). If you have a large number of files to move you might want to use the ``gsutil -m`` option, to perform a multi-threaded/multi-processing move: gsutil -m mv gs://my_bucket/oldprefix gs://my_bucket/newprefix <B>NON-ATOMIC OPERATION</B> Unlike the case with many file systems, the gsutil mv command does not perform a single atomic operation. Rather, it performs a copy from source to destination followed by removing the source for each object. A consequence of this is that, in addition to normal network and operation charges, if you move a Nearline Storage, Coldline Storage, or Archive Storage object, deletion and data retrieval charges apply. See the `documentation <https://cloud.google.com/storage/pricing>`_ for pricing details. <B>OPTIONS</B> All options that are available for the gsutil cp command are also available for the gsutil mv command (except for the -R flag, which is implied by the ``gsutil mv`` command). Please see the `Options section for cp <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for more information. c sx e Zd ZdZejdg d�edeeddde j e j ge j e � � gd� Zejdd d gd d ei d �Z� fdd�Zdd� Z� ZS )� MvCommandz�Implementation of gsutil mv command. Note that there is no atomic rename operation - this command is simply a shorthand for 'cp' followed by 'rm'. �mv)�moveZren�rename� TFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsr r � command_helpzMove/rename objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s& t | j� tddgtd�}t� �|�S )N�storager )�gcloud_command�flag_map)r �sub_optsr r �super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� � /opt/gsutil/gslib/commands/mv.pyr, � s �z!MvCommand.get_gcloud_storage_argsc C s� | j dd� D ]:}t|�}|�� r|�� s2|�� rtd|| j d f ��qdg}| jr`|�d� |�| j � | j j d|| j | j | j| j| jd� dS )z'Command entry point for the mv command.r ���z�You cannot move a source bucket using the mv command. If you meant to move all objects in the bucket, you can use a command like: gsutil mv %s/* %sz-Mz-R�cp)�args�headers�debug� trace_token� user_project�parallel_operations)r5 r � IsCloudUrl�IsBucket� IsProviderr �recursion_requested�append�extend� unparsed_args�command_runner�RunNamedCommandr6 r7 r8 r9 r: )r- Z arg_to_check�urlrA r1 r1 r2 � RunCommand� s( � � zMvCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r r �XML�JSONr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr, rE � __classcell__r1 r1 r/ r2 r ` s4 ��� r N)rI � __future__r r r r � gslib.commandr �gslib.command_argumentr Zgslib.commands.cpr r r �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr �gslib.utils.constantsr �gslib.utils.shim_utilr rK rQ r r1 r1 r1 r2 �<module> s* ��9 __pycache__/rm.cpython-39.pyc 0000644 00000025561 15025166704 0012042 0 ustar 00 a +(Wg@ � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z dd lm Z dd lm Z dd l m Z dd l mZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' dd l(m)Z) dd l*m+Z+ e j,Z-d Z.d e. d Z/d!d"� Z0d#d$� Z1d+d%d&�Z2d'd(� Z3G d)d*� d*e �Z4dS ),zCImplementation of Unix-like rm command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketNotFoundException)�NotEmptyException)�NotFoundException)�ServiceException)�Command)�DecrementFailureCount)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_PREFIX)�NO_URLS_MATCHED_TARGET)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�MetadataMessage)� constants)�parallelism_framework_util)�GetCloudApiInstance)�Retry)�GcloudStorageFlag)�GcloudStorageMap)� StdinIterator)�PreconditionsFromHeadersz7 gsutil rm [-f] [-r] url... gsutil rm [-f] [-r] -I z <B>SYNOPSIS</B> aU <B>DESCRIPTION</B> NOTE: As part of verifying the existence of objects prior to deletion, ``gsutil rm`` makes ``GET`` requests to Cloud Storage for object metadata. These requests incur `network and operations charges <https://cloud.google.com/storage/pricing>`_. The gsutil rm command removes objects and/or buckets. For example, the following command removes the object ``kitten.png``: gsutil rm gs://bucket/kitten.png Use the -r option to specify recursive object deletion. For example, the following command removes gs://bucket/subdir and all objects and subdirectories under it: gsutil rm -r gs://bucket/subdir When working with versioning-enabled buckets, note that the -r option removes all object versions in the subdirectory. To remove only the live version of each object in the subdirectory, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_. The following command removes all versions of all objects in a bucket, and then deletes the bucket: gsutil rm -r gs://bucket To remove all objects and their versions from a bucket without deleting the bucket, use the ``-a`` option: gsutil rm -a gs://bucket/** If you have a large number of objects to remove, use the ``gsutil -m`` option, which enables multi-threading/multi-processing: gsutil -m rm -r gs://my_bucket/subdir You can pass a list of URLs (one per line) to remove on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to remove objects identified by a program, such as: some_program | gsutil -m rm -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. Note that ``gsutil rm`` refuses to remove files from the local file system. For example, this fails: gsutil rm *.txt WARNING: Object removal cannot be undone. Cloud Storage is designed to give developers a high amount of flexibility and control over their data, and Google maintains strict controls over the processing and purging of deleted data. If you have concerns that your application software or your users may at some point erroneously delete or replace data, see `Options for controlling data lifecycles <https://cloud.google.com/storage/docs/control-data-lifecycles>`_ for ways to protect your data from accidental data deletion. <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing multiple objects. If some of the objects could not be removed, gsutil's exit status will be non-zero even if this flag is set. Execution will still halt if an inaccessible bucket is encountered. This option is implicitly set when running "gsutil -m rm ...". -I Causes gsutil to read the list of objects to remove from stdin. This allows you to run a program that generates the list of objects to remove. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents (all objects and subdirectories that it contains) to be removed recursively. If used with a bucket-only URL (like gs://bucket), after deleting objects and subdirectories gsutil deletes the bucket. This option implies the -a option and deletes all object versions. If you only want to delete live object versions, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_ instead of -r. -a Delete all versions of an object. c C sf | j s| j�t|�� t|t�r@| jd7 _| j�t|�� n"t| j|�rTt � n| j d7 _ dS )z9Simple exception handler to allow post-completion status.� N) �continue_on_error�logger�error�str� isinstancer �bucket_not_found_count� _ExceptionMatchesBucketToDelete�bucket_strings_to_deleter �op_failure_count��cls�e� r* � /opt/gsutil/gslib/commands/rm.py�_RemoveExceptionHandler� s r, c C s. t |t�rt|jv s t |t�r&t� n|�dS )z3When removing folders, we don't mind if none exist.N)r" r r �reasonr r r' r* r* r+ � _RemoveFoldersExceptionHandler� s �r. c C s | j ||d� d S )N�� thread_state)� RemoveFunc)r( �name_expansion_resultr0 r* r* r+ �_RemoveFuncWrapper� s r3 c C sB | r>t d }|t|�v r>t|��|�}t|�dko<|d | v S dS )a� Returns True if the exception matches a bucket slated for deletion. A recursive delete call on an empty bucket will raise an exception when listing its objects, but if we plan to delete the bucket that shouldn't result in a user-visible error. Args: bucket_strings_to_delete: Buckets slated for recursive deletion. e: Exception to check. Returns: True if the exception was a no-URLs-matched exception and it matched one of bucket_strings_to_delete, None otherwise. � � r N)r r! �split�len)r% r) �msg�partsr* r* r+ r$ � s r$ c @ s� e Zd ZdZejdg d�edejdddde j e j ge j e � � gd� Zejdg d�dd ei d �Zed dged �ed �ed �ed�ed�d�d�Zdd� Zddd�ZdS )� RmCommandz$Implementation of gsutil rm command.�rm)�del�delete�remover ZafIrRF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRemove objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�-r�-a�-Iz--continue-on-error)rR �-RrS rT �-f)�gcloud_command�flag_mapc sr � j � _d� _d� _� jrv� jD ]T\}}|dkr8d� _q |dkrHd� _q |dkrXd� _q |dksh|dkr d� _d� _q � jr�� jr�td��t� }n� js�td ��� j}d � _ d � _ g }g � _ � j�r d g}|D ]L}t |���� � s��� rΈ �|�j|d �D ] }|�|j� � j �|� q�q�t� j�p(i �� _z�t� j� j� j� j|� j� j� j� j�p\� j d � }d} � j�s�t � j� j� � � |� j� j� jd�} � j t |t � j ddg| d� W nl t�y� } z$t!� j | ��r�t"� n� W Y d} ~ n8d} ~ 0 t#�y } z� j�s � W Y d} ~ n d} ~ 0 0 � j �r0td��� j �rH� j�sHtd��� j�rg } |D ],}t |����$� �rX| �|�%d�d � �qX| �rd� _z<t� j� j� j� j| � j� j� jd�}� j t |t&dd� W n6 t�y } z | j'�(t)��s� W Y d} ~ n d} ~ 0 0 |D ]6�� j�*d�� t+t,ddd �� �fd d ��} | � �q � j �rn� j �rVd nd } td!� j | | f ��d S )"z'Command entry point for the rm command.FrS TrV rT rR rU z&No arguments allowed with the -I flag.z5The rm command (without -I) expects at least one URL.r �id)� bucket_fields)� project_id� all_versionsr N)r\ r[ r& r# )� fail_on_error� shared_attrs�seek_ahead_iteratorz.Encountered non-existent bucket during listingz Some files could not be removed.�*z *_$folder$)r[ r\ )r] �Removing %s...� r )�tries� timeout_secsc s � j j�j�jd� d S )N)�provider)� gsutil_api� DeleteBucket� bucket_name�schemer* ��self�urlr* r+ �BucketDeleteWithRetry~ s z3RmCommand.RunCommand.<locals>.BucketDeleteWithRetry�sr4 z(%d file%s/object%s could not be removed.)-�parallel_operationsr �read_args_from_stdinr\ �sub_opts�recursion_requested�argsr r r& r# r% r �IsBucket� IsProvider�WildcardIterator� IterBuckets�append� storage_urlr �headers� preconditionsr � command_name�debugr rf r[ r �GetSeekAheadGsutilApi�Applyr3 r, r$ r r �IsObject�rstripr. r- � startswithr �infor r )rk �o�unused_a�url_strsZbucket_urls_to_deleterZ �url_str�blr�name_expansion_iteratorr_ r) Zfolder_object_wildcardsrm � plural_strr* rj r+ � RunCommand� s� � � � � � � �zRmCommand.RunCommandNc C s� t | |d�}|j}| j�d|� z"|j|j|j| j|j|j d� W n8 t yz } z | j�d|� t � W Y d }~n d }~0 0 t |j tt�� d�� d S )Nr/ ra )r{ � generationre zCannot find %s)� message_time)r �expanded_storage_urlr r� � DeleteObjectrh � object_namer{ r� ri r r �_PutToQueueWithTimeout� status_queuer �time)rk r2 r0 rf � exp_src_urlr) r* r* r+ r1 � s � �zRmCommand.RemoveFunc)N)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr� r1 r* r* r* r+ r: � sF �� �� r: )N)5r� � __future__r r r r r� �gslib.cloud_apir r r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r r �gslib.name_expansionr r �gslib.storage_urlr �gslib.thread_messager Z gslib.utilsr r � gslib.utils.cloud_api_helperr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Zgslib.utils.system_utilr � gslib.utils.translation_helperr �PutToQueueWithTimeoutr� r� r� r, r. r3 r$ r: r* r* r* r+ �<module> sP ��[ __pycache__/iam.cpython-39.pyc 0000644 00000061147 15025166704 0012172 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) ddl*m+Z+ ddl,m-Z- ddl,m.Z. ddl,m/Z/ dd l,m0Z0 dd l1m2Z3 dd l4m5Z5 dd l6m7Z7 dd l8m9Z9 dd!l8m:Z: dd"l4m;Z; dd#l<m=Z= dd$l<m>Z> dd%l<m?Z? dd&l<m@Z@ dd'l<mAZA dd(l<mBZB dd)lCmDZD dd*lEmFZF dd+lEmGZG d,ZHd-ZId.ZJd/ZKd0ZLd1ZMeHeI�Nd2� eJ�Nd2� d3 ZOd4d2�PeKeLeMg� ZQe$eOeQ�ZRe$eIeK�ZSe$eHeL�ZTe$eJeM�ZUe �Vd5�ZWd6ZXd7d8� ZYd9d:� ZZd;d<� Z[d=d>� Z\d?d@� Z]G dAdB� dBe�Z^dS )Cz8Implementation of IAM policy management command for GCS.� )�absolute_import)�print_function)�division)�unicode_literalsN)�zip)� protojson)� DecodeError)�config)�ArgumentException)�PreconditionException)�ServiceException)�Command)�GetFailureCount)�CommandArgument)� ApiSelector)�CommandException)�&IamChOnResourceWithConditionsException)�CreateHelpText)�LogCommandParams)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�PluralityCheckableIterator)�GetSchemeFromUrlString)�IsKnownUrlScheme)�StorageUrlFromString)� UrlsAreMixOfBucketsAndObjects)�storage_v1_messages)� shim_util)�GetCloudApiInstance)�IAM_POLICY_VERSION)�NO_MAX)� iam_helper)�BindingStringToTuple)� BindingsTuple��DeserializeBindingsTuple)�IsEqualBindings)� PatchBindings��SerializeBindingsTuple)�Retry)�GcloudStorageMap)�GcloudStorageFlagz3 gsutil iam set [-afRr] [-e <etag>] file url ... z gsutil iam get url a� gsutil iam ch [-fRr] binding ... url where each binding is of the form: [-d] ("user"|"serviceAccount"|"domain"|"group"):id:role[,...] [-d] ("allUsers"|"allAuthenticatedUsers"):role[,...] -d ("user"|"serviceAccount"|"domain"|"group"):id -d ("allUsers"|"allAuthenticatedUsers") NOTE: The "iam ch" command does not support changing Cloud IAM policies with bindings that contain conditions. As such, "iam ch" cannot be used to add conditions to a policy or to change the policy of a resource that already contains conditions. See additional details below. NOTE: The "gsutil iam" command does not allow you to add convenience values (projectOwner, projectEditor, projectViewer), but you can remove existing ones. a <B>GET</B> The ``iam get`` command gets the Cloud IAM policy for a bucket or object, which you can save and edit for use with the ``iam set`` command. The following examples save the bucket or object's Cloud IAM policy to a text file: gsutil iam get gs://example > bucket_iam.txt gsutil iam get gs://example/important.txt > object_iam.txt The Cloud IAM policy returned by ``iam get`` includes an etag. The etag is used in the precondition check for ``iam set`` unless you override it using ``iam set -e``. a� <B>SET</B> The ``iam set`` command sets a Cloud IAM policy on one or more buckets or objects, replacing the existing policy on those buckets or objects. For an example of the correct formatting for a Cloud IAM policy, see the output of the ``iam get`` command. You can use the ``iam ch`` command to edit an existing policy, even in the presence of concurrent updates. You can also edit the policy concurrently using the ``-e`` flag to override the Cloud IAM policy's etag. Specifying ``-e`` with an empty string (i.e. ``gsutil iam set -e '' ...``) instructs gsutil to skip the precondition check when setting the Cloud IAM policy. When you set a Cloud IAM policy on a large number of objects, you should use the gsutil ``-m`` option for concurrent processing. The following command applies ``iam.txt`` to all objects in the ``dogs`` bucket: gsutil -m iam set -r iam.txt gs://dogs Note that only object-level operations are parallelized; setting a Cloud IAM policy on a large number of buckets with the ``-m`` flag does not improve performance. <B>SET OPTIONS</B> The ``set`` sub-command has the following options: -R, -r Performs ``iam set`` recursively on all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -a Performs ``iam set`` on all object versions. -e <etag> Performs the precondition check on each object with the specified etag before setting the policy. You can retrieve the policy's etag using ``iam get``. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This option is implicitly set when you use the gsutil ``-m`` option. av <B>CH</B> The ``iam ch`` command incrementally updates Cloud IAM policies. You can specify multiple access grants or removals in a single command. The access changes are applied as a batch to each url in the order in which they appear in the command line arguments. Each access change specifies a principal and a role that is either granted or revoked. You can use gsutil ``-m`` to handle object-level operations in parallel. NOTE: The ``iam ch`` command cannot be used to change the Cloud IAM policy of a resource that contains conditions in its policy bindings. Attempts to do so result in an error. To change the Cloud IAM policy of such a resource, you can perform a read-modify-write operation by saving the policy to a file using ``iam get``, editing the file, and setting the updated policy using ``iam set``. <B>CH EXAMPLES</B> Examples for the ``ch`` sub-command: To grant a single role to a single principal for some targets: gsutil iam ch user:john.doe@example.com:objectCreator gs://ex-bucket To make a bucket's objects publicly readable: gsutil iam ch allUsers:objectViewer gs://ex-bucket To grant multiple bindings to a bucket: gsutil iam ch user:john.doe@example.com:objectCreator \ domain:www.my-domain.org:objectViewer gs://ex-bucket To specify more than one role for a particular principal: gsutil iam ch user:john.doe@example.com:objectCreator,objectViewer \ gs://ex-bucket To specify a custom role for a particular principal: gsutil iam ch user:john.doe@example.com:roles/customRoleName gs://ex-bucket To apply a grant and simultaneously remove a binding to a bucket: gsutil iam ch -d group:readers@example.com:legacyBucketReader \ group:viewers@example.com:objectViewer gs://ex-bucket To remove a user from all roles on a bucket: gsutil iam ch -d user:john.doe@example.com gs://ex-bucket <B>CH OPTIONS</B> The ``ch`` sub-command has the following options: -d Removes roles granted to the specified principal. -R, -r Performs ``iam ch`` recursively to all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This is implicitly set when you invoke the gsutil ``-m`` option. � z a7 Cloud Identity and Access Management (Cloud IAM) allows you to control who has access to the resources in your Google Cloud project. For more information, see `Cloud Identity and Access Management <https://cloud.google.com/storage/docs/access-control/iam>`_. The iam command has three sub-commands: z [a-z]+://.+z�To change the IAM policy of a resource that has bindings containing conditions, perform a read-modify-write operation using "iam get" and "iam set".c C s t | �r|std��d S )Nz/Cannot operate on a mix of buckets and objects.)r r )�urls�recursion_requested� r0 �!/opt/gsutil/gslib/commands/iam.py�*_RaiseErrorIfUrlsAreMixOfBucketsAndObjects� s r2 c C s$ |\}}| j |jdd� |D �|d�S )Nc S s g | ] }t |��qS r0 r$ ��.0�tr0 r0 r1 � <listcomp> � z$_PatchIamWrapper.<locals>.<listcomp>�� thread_state)�PatchIamHelper�expanded_storage_url)�cls� iter_resultr9 Zserialized_bindings_tuples�expansion_resultr0 r0 r1 �_PatchIamWrapper s �r? c C s$ |\}}| j |jt�tj|�|d�S )Nr8 )� SetIamHelperr; r �decode_message�apitools_messages�Policy)r< r= r9 Zserialized_policyr> r0 r0 r1 �_SetIamWrapper s �rD c C s | j �t|�� d S �N��logger�error�str�r< �er0 r0 r1 �_SetIamExceptionHandler s rL c C s | j �t|�� d S rE rF rJ r0 r0 r1 �_PatchIamExceptionHandler s rM c s e Zd ZdZejdededdddej gej e � d�ge � d�e � � ge �� e � � gd�d � Zejdg d d eeeed�d �Zd d� Z� fdd�Zdd� Zdd� Zd/dd�Z� fdd�Zd0dd�Z d1d d �Z d2d d �Z d3d d!�Z d4d"d#�Z e!e"d$d%d&�d5d'd(��Z#d)d*� Z$d+d,� Z%d-d.� Z&� Z'S )6� IamCommandz%Implementation of gsutil iam command.�iam� zafRrd:e:TF� )�get�set�ch) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz9Get, set, or change bucket and/or object IAM permissions.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s | j s|d �� rdS dS d S )Nr �objects�buckets)r/ �IsObject)�self� url_patternsr0 r0 r1 �_get_resource_typeD s z IamCommand._get_resource_typec s� | j �d�| _| jdkrLt| j d ��� r0d}nd}td|ddgi d�}�n.| jd k�r@tg d �td �td �td �td�td�td�d�d�}| �� | j dd � }tt t|��}d}t | j �D ]4\}\}}|dv r�d}q�|dkr�|dkr�d| j |< q�t ||� |�s|d �� �r d|j d< n d|j d< || j d d� | _ n:| jdk�rzt�t�ddd��tjju �rv| j�d � g S t� �|�S ) Nr rR rf rg �storage�get-iam-policy� --format=json)�gcloud_command�flag_maprS )rl N�set-iam-policyrn z--all-versionsz--etagz--etag=z--continue-on-errorz --recursive)�-a�-e� _empty_etag�-f�-R�-rrQ F�rw rv Trs � )rt ry rT �GSUtil�hidden_shim_mode�nonez|The shim maps iam ch commands to several gcloud storage commands, which cannot be determined without running gcloud storage.)�args�pop� sub_commandr rh r+ r, � ParseSubOpts�list�map� enumerate�sub_optsr2 ro r �HIDDEN_SHIM_MODEr rR �DRY_RUNrG �warning�super�get_gcloud_storage_args) ri � command_group�gcloud_storage_map� url_strings� url_objects�recurse�i�flag_key� flag_value�� __class__r0 r1 r� J s` �� �� ����z"IamCommand.get_gcloud_storage_argsc C s: |� � r6d|j }t|j��td��r.|d7 }t|��d S )NzInvalid Cloud URL "%s".z-Rrfzz This resource handle looks like a flag, which must appear before all bindings. See "gsutil help iam ch" for more details.)� IsFileUrl� object_namerS �issubsetr )ri �url� error_msgr0 r0 r1 �_RaiseIfInvalidUrl� s �z IamCommand._RaiseIfInvalidUrlc C s* d| _ d| _g }| jr^| jD ]@\}}|dv r4d| _q |dkrDd| _ q |dkr |�td|�� q g }t| j�}|D ]v}t�|�r�t t |��r�|�|� q�|dkr�z|�tdt |��� W q� t y� t d��Y q�0 qp|�td|�� qp|s�t d��|D ]}|�|� q�ttt|��}t|| j� ||fS )NFrx Tru z-dz?A -d flag is missing an argument specifying bindings to remove.z"Must specify at least one binding.)�continue_on_errorr/ r� �appendr" �iterr} �STORAGE_URI_REGEX�matchr r �next� StopIterationr r� r� r r2 )ri �patch_bindings_tuples�o�aZurl_pattern_strings�it�tokenr� r0 r0 r1 �_GetSettingsAndDiffs� sH � � �z IamCommand._GetSettingsAndDiffsNc C sZ | � |�\}}tj|| �� |tjtjdd�}|jdkrV| jrL| j�|j � n t |j ��|S )NT)�env�input�stderr�stdout�textr ) �._get_full_gcloud_storage_execution_information� subprocess�run�'_get_shim_command_environment_variables�PIPE� returncoder� rG rH r� r )ri r} �stdin�_�command�processr0 r0 r1 �_run_ch_subprocess� s � z IamCommand._run_ch_subprocessc sj | j dkrt� �� S | �� | �� \}}| �|�}g }| jrF|�d� d}|D �]}| �|� |dkr�| � g d�| t |�g �}|j r�d}qNt � |j�}dd� |D �} n t |�g} | D ]�} | � d |d | d g�} | j r�d}q�t � | j�} t�| d �} |D ] \}}t�|�}t| ||�} q�d d� tt�| ��D �| d < | j d |d| dgt j| dd�d�}|j r�d}q�qN|S )NrT rw r rf )rl �lsz--jsonrQ c S s g | ]}|d dkr|d �qS )�type� cloud_objectr� r0 )r4 �resourcer0 r0 r1 r6 � s �z1IamCommand.run_gcloud_storage.<locals>.<listcomp>rl rm rn �bindingsc S s" g | ]\}}|t t|��d ��qS �)�role�members)�sortedr� �r4 �r�mr0 r0 r1 r6 � s � �rq �-T)� sort_keys)r� )r r� �run_gcloud_storager� r� rk r/ r� r� r� rI r� �json�loadsr� r! ZBindingsDictToUpdateDictr' r� �six� iteritems�dumps)ri �bindings_tuples�patternsZ resource_typeZ list_settings� return_codeZ url_patternZ ls_processZ ls_outputr. r� Z get_process�policyr� �is_grant�diff� diff_dictZ set_processr� r0 r1 r� � s^ �� � � � �z IamCommand.run_gcloud_storagec C sR t | |d�}|�� r.|j|j|jddgd�}n |j|j|j|j|jddgd�}|S )a� Gets an IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the global -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Returns: Policy instance. r8 r� �etag)�provider�fields)� generationr� r� )r �IsBucket�GetBucketIamPolicy� bucket_name�scheme�GetObjectIamPolicyr� r� )ri � storage_urlr9 � gsutil_apir� r0 r0 r1 � GetIamHelper s ��zIamCommand.GetIamHelperc C s� | j d }t| �|�jdgd��}|�� r6td| ��|�� rPtd|| jf ��tt |�d j �}| j ||d�}t � t�|��}t j|ddd d �}t|� d S ) z,Gets IAM policy for single bucket or object.r �name)�bucket_listing_fieldsz%s matched no URLszD%s matched more than one URL, which is not allowed by the %s commandr8 T)�,z: rP )r� � separators�indentN)r} r �WildcardIterator�IterAll�IsEmptyr � HasPlurality� command_namer r� � url_stringr� r� r� r �encode_messager� �print)ri r9 �pattern�matchesr� r� � policy_jsonZ policy_strr0 r0 r1 �_GetIam) s, � ���zIamCommand._GetIamc C sJ t | |d�}|�� r*|j|j||jd� n |j|j|j||j|jd� dS )aP Sets IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. policy: A Policy object to set on the bucket / object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Raises: ServiceException passed from the API call if an HTTP error was returned. r8 )r� )r� r� N)r r� �SetBucketIamPolicyr� r� �SetObjectIamPolicyr� r� )ri r� r� r9 r� r0 r0 r1 �_SetIamHelperInternalB s ��z IamCommand._SetIamHelperInternalc C s< z| j |||d� W n" ty6 | jr0d| _n� Y n0 dS )zDHandles the potential exception raised by the internal set function.r8 FN)r� r r� �everything_set_okay)ri r� r� r9 r0 r0 r1 r@ c s zIamCommand.SetIamHelperc C s� z| j |||d� W np ty6 | jr0d| _n� Y nP ty� } z8| jrfd| _d| _| j�|j� n t |j��W Y d}~n d}~0 0 dS )a� Patches an IAM policy for a single, resolved bucket / object URL. The patch is applied by altering the policy from an IAM get request, and setting the new IAM with the specified etag. Because concurrent IAM set requests may alter the etag, we may need to retry this operation several times before success. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. bindings_tuples: A list of BindingsTuple instances. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. r8 FTN) �_PatchIamHelperInternalr r� r� r �$tried_ch_on_resource_with_conditionsrG �debug�messager )ri r� r� r9 rK r0 r0 r1 r: m s � zIamCommand.PatchIamHelper� g �?)�tries� timeout_secsc C s� | j ||d�}|j|j }}|D ]:}|jr d| }|d7 }|d�t�dt ��7 }t|��q t |�} |D ]<\} } t � |�} t � | �} t | | | �}dd� t �|�D �}qht|| �r�| j�d|� d S tj||d�}| j|||d� d S ) Nr8 z"Could not patch IAM policy for %s.r- zfThe resource had conditions present in its IAM policy bindings, which is not supported by "iam ch". %sc S s$ g | ] \}}t jj|t|�d ��qS r� )rB rC �BindingsValueListEntryr� r� r0 r0 r1 r6 � s ��z6IamCommand._PatchIamHelperInternal.<locals>.<listcomp>zNo changes made to %s)r� r� )r� r� r� � condition�join�textwrap�wrap� IAM_CH_CONDITIONS_WORKAROUND_MSGr r� r! ZBindingsMessageToUpdateDictr' r� r� r& rG �inforB rC r� )ri r� r� r9 r� r� r� �bindingr� Z orig_bindingsr� r� Z bindings_dictr� Znew_bindings_dictr0 r0 r1 r� � s6 ��� � z"IamCommand._PatchIamHelperInternalc C s� | � � \}}g }|D ]J\}}g }|D ]&}tjj|d |d d�}|�|� q$|�t||d�� qd| _d| _g } |D ]b} z@| �� r�| j r�d| _ | �| j � q�| � | |� n | �| j � W qt t y� | �| � Y qt0 qt| �rxt| j| j| j| j| | j | j| j�p| jdgd � } t| j| j| �� | | j | jd �} t�d d � |D ��} | jt t | | �t | j | d � | jt � dk M _| j�s�d}| j�r�|d7 }|d� t!�"dt# ��7 }t$|��d S )Nr� r� )r� r� )r� r� TF�*r� �� all_versionsr� r� �r c S s g | ] }t |��qS r0 r( r3 r0 r0 r1 r6 � r7 z(IamCommand._PatchIam.<locals>.<listcomp>�� fail_on_error�seek_ahead_iteratorr z'Some IAM policies could not be patched.r- zjSome resources had conditions present in their IAM policy bindings, which is not supported by "iam ch". %s)%r� rB rC r� r� r# r� r� r� r/ �objectr� r: �AttributeErrorr� r r� r� rG r� r r� �parallel_operationsr �GetSeekAheadGsutilApi� itertools�repeat�Applyr? r rM r r� r� r� r� r )ri Zraw_bindings_tuplesrj r� r� r� Zbindings_messagesr Zbindings_message�threaded_wildcards�surl�name_expansion_iteratorr Z serialized_bindings_tuples_it�msgr0 r0 r1 � _PatchIam� s� � � � � �����zIamCommand._PatchIamc C s� d| _ d| _d| _d}d}| jr~| jD ]V\}}|dv r>d| _q&|dkrNd| _ q&|dkr^d| _q&|dkrtt|�}d}q&| �� q&| jd }| jd d � }z<t|d �� }t� |� � �}W d � n1 s�0 Y W n\ t y� t d | ��Y n@ t �y. } z&| j�d | � t d| ��W Y d } ~ n d } ~ 0 0 |�dg �} |�sN|�dd�}t�| |td��} zt�tj| �}W n$ t�y� t d||f ��Y n0 d| _g } ttt|��} t | | j� | D ]P}t |j � |� � �r| j�r�d|_ | �!|j � n | �"||� n | �!|j � �q�| �r�t#| j$| j| j| j%| | j| j| j �p@| j&dgd� }t'| j$| j| �(� | | j| jd�}t)�*t�+|��}| j,t-t.||�t/| j |d� | jt0� dk M _| j�s�t1d��d S )z7Set IAM policy for given wildcards on the command line.Fry rx Tru rr rs r rQ Nr� z.Specified IAM policy file "%s" does not exist.z'Invalid IAM policy file, ValueError: %sz Invalid IAM policy file "%s".r� r� )r� r� �versionz*Invalid IAM policy file "%s" or etag "%s".r r� r r r z#Some IAM policies could not be set.)2r� r/ r r� rI � RaiseInvalidArgumentExceptionr} �openr� r� �read�IOErrorr � ValueErrorrG r� rR r� r r rA rB rC r r� r� r� r r2 r� r� r� r� r� r@ r r� r� r r r r r r� r rD r rL r r )ri Z force_etagr� r� �argZfile_urlr� �fpr� rK r� r� r Zsurlsr r r Z policy_itr0 r0 r1 �_SetIam s� 0 � " �� � ��zIamCommand._SetIamc C s� | j �d�}| jdd� t| jd� d| _|dkrHt|gd� | �� nN|dkrft|gd� | �� n0|d kr�t|gd� | �� nt d || j f ��dS ) z(Command entry point for the acl command.r T)� check_args)r� FrR )� subcommandsrS rT zBInvalid subcommand "%s" for the %s command. See "gsutil help iam".) r} r~ r� r r� �def_aclr� r r r r� )ri �action_subcommandr0 r0 r1 � RunCommandv s" �zIamCommand.RunCommand)N)N)N)N)N)N)N)(�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSONr �MakeNCloudURLsArgument�MakeNFileURLsArgument� MakeZeroOrMoreCloudURLsArgument� MakeOneOrMoreBindingsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrk r� r� r� r� r� r� r� r� r@ r: r* r r� r r r" � __classcell__r0 r0 r� r1 rN sb ������ 8 1 : ! ! " � /InrN )_r& � __future__r r r r r r� �os�rer� r� r� � six.movesr �apitools.base.protorpcliter Z#apitools.base.protorpclite.messagesr �botor �gslib.cloud_apir r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr Z gslib.metricsr �gslib.name_expansionr r �"gslib.plurality_checkable_iteratorr �gslib.storage_urlr r r r �"gslib.third_party.storage_apitoolsr rB Z gslib.utilsr � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r r! Zgslib.utils.iam_helperr" r# r% r& r' r) �gslib.utils.retry_utilr* �gslib.utils.shim_utilr+ r, � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�_CH_DESCRIPTION�lstripr( r� � _DESCRIPTIONr0 r1 r2 r3 �compiler� r� r2 r? rD rL rM rN r0 r0 r0 r1 �<module> s� +E ��� � __pycache__/cp.cpython-39.pyc 0000644 00000124322 15025166704 0012021 0 ustar 00 a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl m!Z! ddl m"Z" ddl m#Z# ddl$m%Z& ddl'm(Z( ddl'm)Z) ddl'm*Z* ddl+m,Z, ddl-m.Z. dd l-m/Z/ dd l0m1Z1 dd l0m2Z2 dd l0m3Z3 dd l0m4Z4 dd!l0m5Z5 dd"l0m6Z6 dd#l7m8Z8 dd$l7m9Z9 dd%l7m:Z: dd&l7m;Z; dd'l7m<Z< dd(l7m=Z= dd)l>m?Z? dd*l>m@Z@ dd+lAmBZB dd,lAmCZC dd-lAmDZD dd.lEmFZF dd/lEmGZG dd0lHmIZI dd1lHmJZJ d2ZKd3eK ZLd4ZMd5ZNd6ZOd7ZPd8ZQd9ZRd:ZSd;ZTd<ZUd=ZVd>ZWd?ZXd@ZYdA�ZeLeMeNeOePeQeReSeTeUeVeWeXeYg�Z[dBZ\e?dC�e?dD�e?dE�e?dF�e?dG�e?dH�e?dI�e?dJ�e?dK�e?dL�e?dM�e?dN�e?dO�e?dP�e?dQ�e?dR�dS�Z]dTdU� e^e]�_� �dVe?dV�fdWe?dV�fg D �Z`dXdY� ZadbdZd[�Zbd\d]� Zcd^d_� ZdG d`da� dae�ZedS )czCImplementation of Unix-like cp command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)� gcs_json_api)�Command)�CommandArgument)� ApiSelector)�CommandException)�LogPerformanceSummaryParams)�CopyObjectsIterator)�DestinationInfo)�NameExpansionIterator)�%NameExpansionIteratorDestinationTuple)� SeekAheadNameExpansionIterator)�ContainsWildcard)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� cat_helper)� copy_helper)�parallelism_framework_util)�GetCloudApiInstance)�DEBUGLEVEL_DUMP_REQUESTS)�NO_MAX)�CreateCopyHelperOpts)� GetSourceFieldsNeededForCopy)�GZIP_ALL_FILES)�ItemExistsError)�Manifest)�SkipUnsupportedObjectError)�ConvertModeToBase8)�+DeserializeFileAttributesFromObjectMetadata)�InitializePreservePosixData)�POSIXAttributes)�'SerializeFileAttributesToObjectMetadata)� ValidateFilePermissionAccess)�GcloudStorageFlag)�GcloudStorageMap)�GetStreamFromFileUrl)� StdinIterator)�StdinIteratorCls)�NormalizeStorageClass)�RemoveCRLFFromString)�CalculateThroughput)�MakeHumanReadablezw gsutil cp [OPTION]... src_url dst_url gsutil cp [OPTION]... src_url... dst_url gsutil cp [OPTION]... -I dst_url z <B>SYNOPSIS</B> an <B>DESCRIPTION</B> The ``gsutil cp`` command allows you to copy data between your local file system and the cloud, within the cloud, and between cloud storage providers. For example, to upload all text files from the local directory to a bucket, you can run: gsutil cp *.txt gs://my-bucket You can also download data from a bucket. The following command downloads all text files from the top-level of a bucket to your current directory: gsutil cp gs://my-bucket/*.txt . You can use the ``-n`` option to prevent overwriting the content of existing files. The following example downloads text files from a bucket without clobbering the data in your directory: gsutil cp -n gs://my-bucket/*.txt . Use the ``-r`` option to copy an entire directory tree. For example, to upload the directory tree ``dir``: gsutil cp -r dir gs://my-bucket If you have a large number of files to transfer, you can perform a parallel multi-threaded/multi-processing copy using the top-level gsutil ``-m`` option (see "gsutil help options"): gsutil -m cp -r dir gs://my-bucket You can use the ``-I`` option with ``stdin`` to specify a list of URLs to copy, one per line. This allows you to use gsutil in a pipeline to upload or download objects as generated by a program: cat filelist | gsutil -m cp -I gs://my-bucket or: cat filelist | gsutil -m cp -I ./download_dir where the output of ``cat filelist`` is a list of files, cloud URLs, and wildcards of files and cloud URLs. NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand wildcards in ways that can be surprising. You may also encounter issues when attempting to copy files whose names contain wildcard characters. For more details about these issues, see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. ah <B>HOW NAMES ARE CONSTRUCTED</B> The ``gsutil cp`` command attempts to name objects in ways that are consistent with the Linux ``cp`` command. This means that names are constructed depending on whether you're performing a recursive directory copy or copying individually-named objects, or whether you're copying to an existing or non-existent directory. When you perform recursive directory copies, object names are constructed to mirror the source directory structure starting at the point of recursive processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the following command creates the object ``gs://my-bucket/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket In contrast, copying individually-named files results in objects named by the final path component of the source files. For example, assuming again that ``dir1/dir2`` contains ``a/b/c``, the following command creates the object ``gs://my-bucket/c``: gsutil cp dir1/dir2/** gs://my-bucket Note that in the above example, the '**' wildcard matches all names anywhere under ``dir``. The wildcard '*' matches names just one level deep. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. The same rules apply for uploads and downloads: recursive copies of buckets and bucket subdirectories produce a mirrored filename structure, while copying individually or wildcard-named objects produce flatly-named files. In addition, the resulting names depend on whether the destination subdirectory exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory, the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket/subdir In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp`` command creates the object ``gs://my-bucket/subdir/a/b/c``. NOTE: The `Google Cloud Platform Console <https://console.cloud.google.com>`_ creates folders by creating "placeholder" objects that end with a "/" character. gsutil skips these objects when downloading from the cloud to the local file system, because creating a file that ends with a "/" is not allowed on Linux and macOS. We recommend that you only create objects that end with "/" if you don't intend to download such objects using gsutil. a� <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> You can use gsutil to copy to and from subdirectories by using a command like this: gsutil cp -r dir gs://my-bucket/data This causes ``dir`` and all of its files and nested subdirectories to be copied under the specified destination, resulting in objects with names like ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket subdirectories using the following command: gsutil cp -r gs://my-bucket/data dir This causes everything nested under ``gs://my-bucket/data`` to be downloaded into ``dir``, resulting in files with names like ``dir/data/a/b/c``. Copying subdirectories is useful if you want to add data to an existing bucket directory structure over time. It's also useful if you want to parallelize uploads and downloads across multiple machines (potentially reducing overall transfer time compared with running ``gsutil -m cp`` on one machine). For example, if your bucket contains this structure: gs://my-bucket/data/result_set_01/ gs://my-bucket/data/result_set_02/ ... gs://my-bucket/data/result_set_99/ you can perform concurrent downloads across 3 machines by running these commands on each machine, respectively: gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir Note that ``dir`` could be a local directory on each machine, or a directory mounted off of a shared file server. The performance of the latter depends on several factors, so we recommend experimenting to find out what works best for your computing environment. a� <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast, when you download data from the cloud, it ends up in a file with no associated metadata, unless you have some way to keep or re-create that metadata. Copies spanning locations and/or storage classes cause data to be rewritten in the cloud, which may take some time (but is still faster than downloading and re-uploading). Such operations can be resumed with the same command if they are interrupted, so long as the command parameters are identical. Note that by default, the gsutil ``cp`` command does not copy the object ACL to the new object, and instead uses the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the ``-p`` option. When copying in the cloud, if the destination bucket has Object Versioning enabled, by default ``gsutil cp`` copies only live versions of the source object. For example, the following command causes only the single live version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there are noncurrent versions of ``gs://bucket1/obj``: gsutil cp gs://bucket1/obj gs://bucket2 To also copy noncurrent versions, use the ``-A`` flag: gsutil cp -A gs://bucket1/obj gs://bucket2 The top-level gsutil ``-m`` flag is not allowed when using the ``cp -A`` flag. z� <B>CHECKSUM VALIDATION</B> gsutil automatically performs checksum validation for copies to and from Cloud Storage. For more information, see `Hashes and ETags <https://cloud.google.com/storage/docs/hashes-etags#cli>`_. a <B>RETRY HANDLING</B> The ``cp`` command retries when failures occur, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the ``cp`` command skips that object and moves on. If any failures were not successfully retried by the end of the copy run, the ``cp`` command reports the number of failures and exits with a non-zero status. For details about gsutil's overall retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. a� <B>RESUMABLE TRANSFERS</B> gsutil automatically resumes interrupted downloads and interrupted `resumable uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_, except when performing streaming transfers. In the case of an interrupted download, a partially downloaded temporary file is visible in the destination directory with the suffix ``_.gstmp`` in its name. Upon completion, the original file is deleted and replaced with the downloaded contents. Resumable transfers store state information in files under ~/.gsutil, named by the destination object or file. See "gsutil help prod" for details on using resumable transfers in production. a� <B>STREAMING TRANSFERS</B> Use '-' in place of src_url or dst_url to perform a `streaming transfer <https://cloud.google.com/storage/docs/streaming>`_. Streaming uploads using the `JSON API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered in memory part-way back into the file and can thus sometimes resume in the event of network or service problems. gsutil does not support resuming streaming uploads using the XML API or resuming streaming downloads for either JSON or XML. If you have a large amount of data to transfer in these cases, we recommend that you write the data to a local file and copy that file rather than streaming it. a< <B>SLICED OBJECT DOWNLOADS</B> gsutil can automatically use ranged ``GET`` requests to perform downloads in parallel for large files being downloaded from Cloud Storage. See `sliced object download documentation <https://cloud.google.com/storage/docs/sliced-object-downloads>`_ for a complete discussion. a� <B>PARALLEL COMPOSITE UPLOADS</B> gsutil can automatically use `object composition <https://cloud.google.com/storage/docs/composite-objects>`_ to perform uploads in parallel for large, local files being uploaded to Cloud Storage. See the `parallel composite uploads documentation <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a complete discussion. am <B>CHANGING TEMP DIRECTORIES</B> gsutil writes data to a temporary directory in several cases: - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options) - when decompressing data being downloaded (for example, when the data has ``Content-Encoding:gzip`` as a result of being uploaded using gsutil cp -z or gsutil cp -Z) - when running integration tests using the gsutil test command In these cases, it's possible the temporary file location on your system that gsutil selects by default may not have enough space. If gsutil runs out of space during one of these operations (for example, raising "CommandException: Inadequate temp space available to compress <your file>" during a ``gsutil cp -z`` operation), you can change where it writes these temp files by setting the TMPDIR environment variable. On Linux and macOS, you can set the variable as follows: TMPDIR=/some/directory gsutil cp ... You can also add this line to your ~/.bashrc file and restart the shell before running gsutil: export TMPDIR=/some/directory On Windows 7, you can change the TMPDIR environment variable from Start -> Computer -> System -> Advanced System Settings -> Environment Variables. You need to reboot after making this change for it to take effect. Rebooting is not necessary after running the export command on Linux and macOS. a <B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B> Please see the section about OS-specific file types in "gsutil help rsync". While that section refers to the ``rsync`` command, analogous points apply to the ``cp`` command. aT2 <B>OPTIONS</B> -a predef_acl Applies the specific predefined ACL to uploaded objects. See "gsutil help acls" for further details. -A Copy all source versions from a source bucket or folder. If not set, only the live version of each source object is copied. NOTE: This option is only useful when the destination bucket has Object Versioning enabled. Additionally, the generation numbers of copied versions do not necessarily match the order of the original generation numbers. -c If an error occurs, continue attempting to copy the remaining files. If any copies are unsuccessful, gsutil's exit status is non-zero, even if this flag is set. This option is implicitly set when running ``gsutil -m cp...``. NOTE: ``-c`` only applies to the actual copying operation. If an error, such as ``invalid Unicode file name``, occurs while iterating over the files in the local directory, gsutil prints an error message and aborts. -D Copy in "daisy chain" mode, which means copying between two buckets by first downloading to the machine where gsutil is run, then uploading to the destination bucket. The default mode is a "copy in the cloud," where data is copied between two buckets without uploading or downloading. During a "copy in the cloud," a source composite object remains composite at its destination. However, you can use "daisy chain" mode to change a composite object into a non-composite object. For example: gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp gsutil mv gs://bucket/obj_tmp gs://bucket/obj NOTE: "Daisy chain" mode is automatically used when copying between providers: for example, when copying data from Cloud Storage to another provider. -e Exclude symlinks. When specified, symbolic links are not copied. -I Use ``stdin`` to specify a list of files or objects to copy. You can use gsutil in a pipeline to upload or download objects as generated by a program. For example: cat filelist | gsutil -m cp -I gs://my-bucket where the output of ``cat filelist`` is a one-per-line list of files, cloud URLs, and wildcards of files and cloud URLs. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the ``-j`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files. This also saves network bandwidth while leaving the data uncompressed in Cloud Storage. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the ``Content-Type`` and name of the original files. Note that if you want to use the ``-m`` `top-level option <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_ to parallelize copies along with the ``-j/-J`` options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. You can change this compression buffer size to a higher limit. For example: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \ -m cp -j html,txt -r /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the ``-j`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in longer uploads. -L <file> Outputs a manifest log file with detailed information about each item that was copied. This manifest contains the following information for each item: - Source path. - Destination path. - Source size. - Bytes transferred. - MD5 hash. - Transfer start time and date in UTC and ISO 8601 format. - Transfer completion time and date in UTC and ISO 8601 format. - Upload id, if a resumable upload was performed. - Final result of the attempted transfer, either success or failure. - Failure details, if any. If the log file already exists, gsutil uses the file as an input to the copy process, and appends log items to the existing file. Objects that are marked in the existing log file as having been successfully copied or skipped are ignored. Objects without entries are copied and ones previously marked as unsuccessful are retried. This option can be used in conjunction with the ``-c`` option to build a script that copies a large number of objects reliably, using a bash script like the following: until gsutil cp -c -L cp.log -r ./dir gs://bucket; do sleep 1 done The -c option enables copying to continue after failures occur, and the -L option allows gsutil to pick up where it left off without duplicating work. The loop continues running as long as gsutil exits with a non-zero status. A non-zero status indicates there was at least one failure during the copy operation. NOTE: If you are synchronizing the contents of a directory and a bucket, or the contents of two buckets, see "gsutil help rsync". -n No-clobber. When specified, existing files or objects at the destination are not replaced. Any items that are skipped by this option are reported as skipped. gsutil performs an additional GET request to check if an item exists before attempting to upload the data. This saves gsutil from retransmitting data, but the additional HTTP requests may make small object transfers slower and more expensive. -p Preserves ACLs when copying in the cloud. Note that this option has performance and cost implications only when using the XML API, as the XML API requires separate HTTP calls for interacting with ACLs. You can mitigate this performance issue using ``gsutil -m cp`` to perform parallel copying. Note that this option only works if you have OWNER access to all objects that are copied. If you want all objects in the destination bucket to end up with the same ACL, you can avoid these performance issues by setting a default object ACL on that bucket instead of using ``cp -p``. See "gsutil help defacl". Note that it's not valid to specify both the ``-a`` and ``-p`` options together. -P Enables POSIX attributes to be preserved when objects are copied. ``gsutil cp`` copies fields provided by ``stat``. These fields are the user ID of the owner, the group ID of the owning group, the mode or permissions of the file, and the access and modification time of the file. For downloads, these attributes are only set if the source objects were uploaded with this flag enabled. On Windows, this flag only sets and restores access time and modification time. This is because Windows doesn't support POSIX uid/gid/mode. -R, -r The ``-R`` and ``-r`` options are synonymous. They enable directories, buckets, and bucket subdirectories to be copied recursively. If you don't use this option for an upload, gsutil copies objects it finds and skips directories. Similarly, if you don't specify this option for a download, gsutil copies objects at the current bucket directory level and skips subdirectories. -s <class> Specifies the storage class of the destination object. If not specified, the default storage class of the destination bucket is used. This option is not valid for copying to non-cloud destinations. -U Skips objects with unsupported object types instead of failing. Unsupported object types include Amazon S3 objects in the GLACIER storage class. -v Prints the version-specific URL for each uploaded object. You can use these URLs to safely make concurrent upload requests, because Cloud Storage refuses to perform an update if the current object version doesn't match the version-specific URL. See `generation numbers <https://cloud.google.com/storage/docs/metadata#generation-number>`_ for more details. -z <ext,...> Applies gzip content-encoding to any file upload whose extension matches the ``-z`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files, because it reduces network bandwidth and storage sizes. This can both improve performance and reduce costs. When you specify the ``-z`` option, the data from your files is compressed before it is uploaded, but your actual files are left uncompressed on the local disk. The uploaded objects retain the ``Content-Type`` and name of the original files, but have their ``Content-Encoding`` metadata set to ``gzip`` to indicate that the object data stored are compressed on the Cloud Storage servers and have their ``Cache-Control`` metadata set to ``no-transform``. For example, the following command: gsutil cp -z html \ cattypes.html tabby.jpeg gs://mycats does the following: - The ``cp`` command uploads the files ``cattypes.html`` and ``tabby.jpeg`` to the bucket ``gs://mycats``. - Based on the file extensions, gsutil sets the ``Content-Type`` of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to ``image/jpeg``. - The ``-z`` option compresses the data in the file ``cattypes.html``. - The ``-z`` option also sets the ``Content-Encoding`` for ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for ``cattypes.html`` to ``no-transform``. Because the ``-z/-Z`` options compress data prior to upload, they are not subject to the same compression buffer bottleneck that can affect the ``-j/-J`` options. Note that if you download an object with ``Content-Encoding:gzip``, gsutil decompresses the content before writing the local file. -Z Applies gzip content-encoding to file uploads. This option works like the ``-z`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in files taking up more space in the cloud than they would if left uncompressed. --stet If the STET binary can be found in boto or PATH, cp will use the split-trust encryption tool for end-to-end encryption. z za:AcDeIL:MNnpPrRs:tUvz:Zj:Jz--all-versionsz--predefined-aclz--continue-on-errorz --daisy-chainz--ignore-symlinksz--read-paths-from-stdinz--gzip-in-flight-allz--gzip-in-flightz--manifest-pathz --no-clobberz--preserve-posixz--preserve-aclz--storage-classz--print-created-messagez--gzip-local-allz --gzip-local)�-A�-a�-c�-D�-e�-I�-J�-j�-L�-n�-P�-p�-s�-v�-Z�-zc C s i | ]\}}||�qS � rA )�.0�k�vrA rA � /opt/gsutil/gslib/commands/cp.py� <dictcomp>� s rF �-r�-Rc C s` d}t | �D ] \}\}}|dkr |} q*q |dur\| | d }| | d tj�||�f| |< dS )aZ Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case. The camel-case-snake-case difference is related to gcloud primarily using JSON API rather than the XML API. Predefined ACLs are also called "canned ACLs". Args: sub_opts: List of pairs representing flag keys and values, e.g. [('a', 'public-read')] Nr2 � r )� enumerater �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION�get)�sub_optsZpredefined_acl_idx�irC �_Zold_predefined_aclrA rA rE �'ShimTranslatePredefinedAclSubOptForCopy� s ��rP c C s | j ||| jd� d S )N)� thread_state�preserve_posix)�CopyFunc�preserve_posix_attrs)�cls�argsrQ rA rA rE �_CopyFuncWrapper� s �rW c C s4 | j �t|�� | jd7 _| j �dt�� � dS )�9Simple exception handler to allow post-completion status.rI z* Encountered exception while copying: %s N)�logger�error�str�op_failure_count�debug� traceback� format_exc�rU �erA rA rE �_CopyExceptionHandler� s �rb c C s | j �t|�� dS )rX N)rY rZ r[ r` rA rA rE �_RmExceptionHandler� s rc c s� e Zd ZdZejddgedeeddde j e j ge j dd ge � � gd � Zejddgd d ei d �Z� fdd�Zddd�Zdd� Zdd� Zdd� Z� ZS )� CpCommandaR Implementation of gsutil cp command. Note that CpCommand is run for both gsutil cp and gsutil mv. The latter happens by MvCommand calling CpCommand and passing the hidden (undocumented) -M option. This allows the copy and remove needed for each mv to run together (rather than first running all the cp's and then all the rm's, as we originally had implemented), which in turn avoids the following problem with removing the wrong objects: starting with a bucket containing only the object gs://bucket/obj, say the user does: gsutil mv gs://bucket/* gs://bucket/d.txt If we ran all the cp's and then all the rm's and we didn't expand the wildcard first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, and the rm command would then remove that object. In the implementation prior to gsutil release 3.12 we avoided this by building a list of objects to process and then running the copies and then the removes; but building the list up front limits scalability (compared with the current approach of processing the bucket listing iterator on the fly). �cp�copyrI TFr Zstetztestcallbackfile=) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�supported_private_args�argparse_arguments� command_helpzCopy files and objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s2 | j �d� t| j� tddgtd�}t� �|�S )Nz�Unlike pure gsutil, this shim won't run composite uploads and sliced downloads in parallel by default. Use the -m flag to enable parallelism (i.e. "gsutil -m cp ...").�storagere )�gcloud_command�flag_map)rY �warnrP rM r) �CP_SHIM_FLAG_MAP�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__rA rE r� � s � �z!CpCommand.get_gcloud_storage_argsNc C sV t | |d�}t�� }|jr d}n| j}|j}|j}|j} |j} |� � r^|� � r^t d||f ��|rz|� � rz|� � rzt d��| jr�|� � r�|� � r�t d��| r�t�|j|j|� t|�r�dS |jr�| j�|j�r�dS |jr�|jr�d| _|j� � �rRtj�|jj��sR| �rRzt� |jj� W n6 t �yP } z | j t j k�r<� W Y d} ~ n d} ~ 0 0 tj ||| | |j!|j|j| j|d� } t�"|| �} t�#|| � t�$|| ��r�t d ||| f ��| � � �r�| �%� �r�t d || f ��| � � �s�|j&�r�t d | ��d} |j'�r t(�)t*j+|j'�} |� � �r�|�r�| �s*t*�+� } t�,|j�\ }}}}}}}}}}t-|�}t.|||||d �}t*j+j/g d �}t0|||d� || _1| �r�| � � �r�t2| |j�}|j3j4}t5|j|j6|j7|d�\}}|�r�|�s�t8�9� �:|� t d��d}�z&|j�r| j�;|j| j� | j�r�|jj�r�| � � �r�tj�<|jj�}tj�=|tj�<| j�g��>|��s�| j?�@d||f � |j�r�| j�A|jddd� W dS tjB| j?|| || tC| d| jD| j| jE| jF|| jGd�\}}}}|j�r�|�r�| j�H|jd|� | j�A|j|d� |jI�r| j?�Jd|� W �n� tK�yX d| } | j?�J| � |j�rR| j�A|jdd| � Y �n� tL�y� } zFd|j| jMf } | j?�J| � |j�r�| j�A|jdd| � W Y d} ~ �nnd} ~ 0 tjN�y� } z"| j?�@d|| f � W Y d} ~ �n2d} ~ 0 tO�y� } z�|jP�rHt�Q| ��rHd| } | j?�J| � |j�r�| j�A|jdd| � nv| jR�r�d |tS| �f } | jTd 7 _T| j?�U| � |j�r�| j�A|jdd tV| �� n"|j�r�| j�A|jdd tS| �� � W Y d} ~ nRd} ~ 0 0 |j�r | j?�Jd |� |� � �r|jW|jX|j|jY|jZd � n t�[|j� | j\� | j]|7 _]W d � n1 �sH0 Y dS )!z@Worker function for performing the actual copy (and rm, for mv).)rQ �mvz<The %s command does not allow provider-only source URLs (%s)z/Cannot preserve POSIX attributes with a stream.z8Cannot upload from a stream when using gsutil -m option.NT)rR z,%s: "%s" and "%s" are the same file - abort.zP%s: a version-specific URL (%s) cannot be the destination for gsutil cp - abort.z<Cannot specify storage class for a non-cloud destination: %s)�atime�mtime�uid�gid�mode)�additionalProperties)r� r� r� zPThis sync will orphan file(s), please fix their permissions before trying again.r ziSkipping copy of source URL %s because it would be copied outside the expected destination directory: %s.�skipz4Would have copied outside the destination directory.)�src_obj_metadataZallow_splitting�headers�manifest� gzip_encoded� gzip_extsrR �use_stet�md5�OKz Created: %szSkipping existing item: %sz0Skipping item %s with unsupported object type %sz�Skipping copy of source URL %s because destination URL %s is already being copied by another gsutil process or thread (did you specify the same source URL twice?) zRejected (noclobber): %szError copying %s: %srI rZ zRemoving %s...)� generation�provider)^r r �GetCopyHelperOpts� perform_mv� command_name�source_storage_url�expanded_storage_url�names_container�is_multi_source_request� IsCloudUrl� IsProviderr � IsFileUrl�IsStream�parallel_operationsZInsistDstUrlNamesContainer� exp_dst_url�have_existing_dst_containerr � use_manifestr� Z WasSuccessful� url_string�recursion_requested�os�path�exists� object_name�makedirs�OSError�errno�EEXISTZConstructDstUrl�!is_multi_top_level_source_requestZFixWindowsNamingZCheckForDirFileConflictZ SrcDstSame� HasGeneration�dest_storage_class�expanded_resultr � JsonToMessage�apitools_messages�Object�statr" r% � MetadataValuer& �metadatar# r� � permissionsr' r� r� �logging� getLogger�criticalZ Initialize�abspath� commonpath� startswithrY r} Z SetResultZ PerformCopyrb r� r� r� r� �Set� print_ver�infor r! Zunsupported_typeZFileConcurrencySkipError� Exception� no_clobberZIsNoClobberServerException�continue_on_errorr[ r\ rZ r. � DeleteObject� bucket_namer� �scheme�unlink� stats_lock�total_bytes_transferred) r� Zcopy_object_inforQ rR � gsutil_api�copy_helper_optsZcmd_name�src_url� exp_src_urlZsrc_url_names_containerZhave_multiple_srcsra �dst_urlr� r� rO r� r� r� r� Z posix_attrs�custom_metadata�valid�err�bytes_transferred� containerZ result_urlr� �messagerA rA rE rS � s� ���� � ��� ���������� �� ������ � *��� � � zCpCommand.CopyFuncc c s� t �� }t||�D ]�\}}t j|| j| j| jd�\}}tt| j | j | j| j|| j pV|j | j| j | j| jpl| jt|�� |j|j| j|j | jd�d� t||��}| jp�|�� | _| jp�|�� | _| j� |j � t � | j |�| _ |V qd S )N)rY )rR Z delete_source�file_size_will_change)� project_id� all_versions�ignore_symlinksr� �bucket_listing_fields)!r r� �zipZExpandUrlToSingleBlrr� r� rY r r r� r] r� r� r� �exclude_symlinksr� r� r r� �skip_unsupported_objects� preserve_aclrT r� r � has_file_dstr� � has_cloud_dst�provider_types�addr� � itertools�chain�combined_src_urls) r� Zsrc_url_strs_iter� dst_url_strsr� � src_url_str� dst_url_strr� r� Z!name_expansion_iterator_dst_tuplerA rA rE �/_ConstructNameExpansionIteratorDstTupleIterator� sN �� ����z9CpCommand._ConstructNameExpansionIteratorDstTupleIteratorc C sh | � � }d| _t| jd �}|�� rz|jdks6|�� rz| jrDtd��|�� rXt |dd�nd }t � | �j | jd d� |d�S |j r�t| j�dkr�td ��t� g}n&t| j�d k r�td ��| jd d� g}| jd g}g | _d | _d | _t� | _t| �||�|j�}| jd d d d d �\}}t�|| j|| � d } |j �slt | j | j | � � | j| j �pV|j!| j"| j#| j$| j%d� } t&�'� | _(d| _)t*�*� } d} | j+t,|t-| | j. | d� | j/� d| j� t*�*� } | | | _0t1| j| j0�| _2t3| j| j| j2| j| j0| j4|jt5| j�d� | j t6k�r4| jdk�r4| j/�7d| j| j0t8| j2�� | j)�rd| j)dk�rLdnd} tdj9| j)| d���dS )Nr ����-z?Cannot preserve POSIX attributes with a stream or a named pipe.�wb)r� )� cat_out_fdrI z.Source URLs cannot be specified with -I option� z+Wrong number of arguments for "cp" command.F)� process_count� thread_count� parallel_operations_override�print_macos_warning)r� r� r� r� )r\ r� )� fail_on_error�seek_ahead_iteratorztotal_bytes_transferred: %d)r� r� �avg_throughputr� �total_elapsed_time�uses_fan�is_daisy_chainr� z;Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)�s� z5{count} file{pl}/object{pl} could not be transferred.)�count�pl):� _ParseOptsr� r rV r� r� �IsFiforT r r* r � CatHelper� CatUrlStrings�read_args_from_stdin�lenr, r� r� r� �setr� r r� � daisy_chain�_GetProcessAndThreadCountr Z.TriggerReauthForDestinationProviderIfNecessaryr� r r� r] �GetSeekAheadGsutilApir� r� r� r� r� r� r � CreateLockr� r\ �time�ApplyrW rb r� rY r� r/ Ztotal_bytes_per_secondr r� �listr r� r0 �format)r� r� r� r� Z src_url_strsr� �name_expansion_iteratorr� r� r� � start_time� shared_attrs�end_timeZ plural_strrA rA rE � RunCommand s� ��� ��� � � �� �� ��zCpCommand.RunCommandc C s� d}d| _ d}d| _d}d}d}d}d}d| _d }d | _d| _d| _d} d} d } d } d } d }d| _d | _| j�r | jD �]�\}}|dkr�|}d| _|dkr�d| _|dkr�d| _qv|dkr�d}qv|dkr�d| _ qv|dkr�|} qv|d kr�d}qv|d k�rd} d d � |� d �D �} qv|dk�r"d} t } qv|dk�r<d}t |�| _qv|dk�rLd}qv|dk�r\d}qv|dk�rld}qv|dk�r�d| _t � qv|dk�s�|dk�r�d| _ qv|dk�r�t|�}qv|dk�r�d| _qv|dk�r�d}qv|dk�r�d} dd � |� d �D �} qv|dk�rd} t } qv|d krvd| _qv|�r2|�r2td ��| j�rJ| j�rJtd ��| �r^| �r^td ��| �r�| �r�| �rztd ��ntd!��| �p�| | _| | _t||||||||| j| |d"� S )#NFr2 Tr1 r3 r4 r5 z--testcallbackfiler6 r8 c S s g | ] }|� � �qS rA ��strip�rB �xrA rA rE � <listcomp>� � z(CpCommand._ParseOpts.<locals>.<listcomp>�,r7 r9 z-Mr: r< r; rG rH r= z-Ur> r@ c S s g | ] }|� � �qS rA r r rA rA rE r � r r? z--stetz:Specifying both the -p and -a options together is invalid.z�The gsutil -m option is not supported with the cp -A flag, to ensure that object version ordering is preserved. Please re-run the command without the -m option.z@Specifying both the -j/-J and -z/-Z options together is invalid.z:Specifying both the -j and -J options together is invalid.z:Specifying both the -z and -Z options together is invalid.) r� r� r r r� r� r� � canned_aclr� �test_callback_filer� )r� r� rT �cannedr� r� r� r� rM �splitr r r$ r� r- r r� r� r� r )r� r� r� r r r� r� r� r r� Z gzip_localZ gzip_arg_extsZ gzip_arg_allr r� �o�arA rA rE r� � s� �� � �� �zCpCommand._ParseOpts)NF)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr � CP_SUB_ARGSr �XML�JSONr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� rS r� r r� � __classcell__rA rA r� rE rd � s> ��� i@xrd )N)fr# � __future__r r r r r� r� r� r� r r^ �apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �gslib.name_expansionr r r r r �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr r r � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r Zgslib.utils.copy_helperr r r r r r! Zgslib.utils.posix_utilr" r# r$ r% r& r' �gslib.utils.shim_utilr( r) Zgslib.utils.system_utilr* r+ r, Zgslib.utils.text_utilr- r. �gslib.utils.unit_utilr/ r0 r% Z_SYNOPSIS_TEXTZ_DESCRIPTION_TEXTZ_NAME_CONSTRUCTION_TEXTZ_SUBDIRECTORIES_TEXTZ_COPY_IN_CLOUD_TEXTZ_CHECKSUM_VALIDATION_TEXTZ_RETRY_HANDLING_TEXTZ_RESUMABLE_TRANSFERS_TEXTZ_STREAMING_TRANSFERS_TEXTZ _SLICED_OBJECT_DOWNLOADS_TEXTZ _PARALLEL_COMPOSITE_UPLOADS_TEXTZ _CHANGING_TEMP_DIRECTORIES_TEXTZ_COPYING_SPECIAL_FILES_TEXTZ _OPTIONS_TEXT�joinr, r& ZCP_AND_MV_SHIM_FLAG_MAPr �itemsr~ rP rW rb rc rd rA rA rA rE �<module> s� �32)% g�� �� __pycache__/cat.cpython-39.pyc 0000644 00000010255 15025166704 0012165 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ejr�eZdZde d ZG dd� de �ZdS )zDImplementation of Unix-like cat command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)�CommandArgument)� ApiSelector)�CommandException)� cat_helper)� constants)�GcloudStorageFlag)�GcloudStorageMapz gsutil cat [-h] url... z <B>SYNOPSIS</B> aJ <B>DESCRIPTION</B> The cat command outputs the contents of one or more URLs to stdout. While the cat command does not compute a checksum, it is otherwise equivalent to doing: gsutil cp url... - (The final '-' causes gsutil to stream the output to stdout.) WARNING: The gsutil cat command does not compute a checksum of the downloaded data. Therefore, we recommend that users either perform their own validation of the output of gsutil cat or use gsutil cp or rsync (both of which perform integrity checking automatically). <B>OPTIONS</B> -h Prints short header for each object. For example: gsutil cat -h gs://bucket/meeting_notes/2012_Feb/*.txt This would print a header with the object name before the contents of each text object that matched the wildcard. -r range Causes gsutil to output just the specified byte range of the object. Ranges can be of these forms: start-end (e.g., -r 256-5939) start- (e.g., -r 256-) -numbytes (e.g., -r -5) where offsets start at 0, start-end means to return bytes start through end (inclusive), start- means to return bytes start through the end of the object, and -numbytes means to return the last numbytes of the object. For example: gsutil cat -r 256-939 gs://bucket/object returns bytes 256 through 939, while: gsutil cat -r -5 gs://bucket/object returns the final 5 bytes of the object. c @ s� e Zd ZdZejdg edejdddde j e j ge j e � � gd� Zejdg dd ei d �Zed dged �ed �d�d�Zdd� ZdS )� CatCommandz%Implementation of gsutil cat command.�cat� zhr:Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz$Concatenate object content to stdout)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storagez-d�-r)�-hr$ )�gcloud_command�flag_mapc C s� d}d}d}d}| j r�| j D ]�\}}|dkr2d}q |dkr�|�� }|dkrLq t�d�}|�|�}|sptd | ��|�d �r�t|�d ��}|�d �r�t|�d ��}|�d �r�t|�d ��}q | �� q t � | �j | j |||d �S )z(Command entry point for the cat command.FNr r% Tr$ �-z;^(?P<start>[0-9]+)-(?P<end>[0-9]*)$|^(?P<endslice>-[0-9]+)$zInvalid range (%s)�start�endZendslice)� show_header� start_byte�end_byte) �sub_opts�strip�re�compile�matchr �group�long� RaiseInvalidArgumentExceptionr Z CatHelperZ CatUrlStrings�args) �selfr+ Z request_ranger, r- �o�aZ range_matcherZ range_match� r: �!/opt/gsutil/gslib/commands/cat.py� RunCommand� s< � �zCatCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr< r: r: r: r; r ] s< �� �� r ) r@ � __future__r r r r r0 �six� gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.utilsr r �gslib.utils.shim_utilr r �PY3�intr4 rB rI r r: r: r: r; �<module> s. ��2 __pycache__/notification.cpython-39.pyc 0000644 00000062743 15025166704 0014115 0 ustar 00 a +(WgŇ � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z dd l mZ dd l mZ dd l mZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( ddl'm)Z) dZ*dZ+d Z,d Z-d Z.e,e+�/d � e*�/d � e- e.�/d � d Z0d Z1d!Z2d"Z3d#Z4d$Z5d%e3 e1 e2 d& e4 e5 d'j6e%j7d(� Z8d)Z9ee0e8�Z:ee,e3�Z;ee*e1�Z<ee+e2�Z=ee-e4�Z>ee.e5�Z?d*d+d,�Z@G d-d.� d.e�ZAdS )/z8This module provides the notification command to gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�datetime)�metrics)�AccessDeniedException)�NotFoundException)� PublishPermissionDeniedException)�Command)�NO_MAX)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText)�PopulateProjectId)� PubsubApi)�StorageUrlFromString)�Binding)� copy_helper)� shim_util)�GcloudStorageFlag)�GcloudStorageMapz2 gsutil notification list gs://<bucket_name>... zO gsutil notification delete (<notificationConfigName>|gs://<bucket_name>)... z� gsutil notification create -f (json|none) [-p <prefix>] [-t <topic>] \ [-m <key>:<value>]... [-e <eventType>]... gs://<bucket_name> zW gsutil notification watchbucket [-i <id>] [-t <token>] <app_url> gs://<bucket_name> z> gsutil notification stopchannel <channel_id> <resource_id> � a� <B>LIST</B> The list sub-command provides a list of notification configs belonging to a given bucket. The listed name of each notification config can be used with the delete sub-command to delete that specific notification config. For listing Object Change Notifications instead of Cloud Pub/Sub notification subscription configs, add a -o flag. <B>LIST EXAMPLES</B> Fetch the list of notification configs for the bucket example-bucket: gsutil notification list gs://example-bucket The same as above, but for Object Change Notifications instead of Cloud Pub/Sub notification subscription configs: gsutil notification list -o gs://example-bucket Fetch the notification configs in all buckets matching a wildcard: gsutil notification list gs://example-* Fetch all of the notification configs for buckets in the default project: gsutil notification list gs://* a� <B>DELETE</B> The delete sub-command deletes notification configs from a bucket. If a notification config name is passed as a parameter, that notification config alone is deleted. If a bucket name is passed, all notification configs associated with that bucket are deleted. Cloud Pub/Sub topics associated with this notification config are not deleted by this command. Those must be deleted separately, for example with the gcloud command `gcloud beta pubsub topics delete`. Object Change Notification subscriptions cannot be deleted with this command. For that, see the command `gsutil notification stopchannel`. <B>DELETE EXAMPLES</B> Delete a single notification config (with ID 3) in the bucket example-bucket: gsutil notification delete projects/_/buckets/example-bucket/notificationConfigs/3 Delete all notification configs in the bucket example-bucket: gsutil notification delete gs://example-bucket a� <B>CREATE</B> The create sub-command creates a notification config on a bucket, establishing a flow of event notifications from Cloud Storage to a Cloud Pub/Sub topic. As part of creating this flow, the create command also verifies that the destination Cloud Pub/Sub topic exists, creating it if necessary, and verifies that the Cloud Storage bucket has permission to publish events to that topic, granting the permission if necessary. If a destination Cloud Pub/Sub topic is not specified with the -t flag, Cloud Storage chooses a topic name in the default project whose ID is the same as the bucket name. For example, if the default project ID specified is 'default-project' and the bucket being configured is gs://example-bucket, the create command uses the Cloud Pub/Sub topic "projects/default-project/topics/example-bucket". In order to enable notifications, your project's `Cloud Storage service agent <https://cloud.google.com/storage/docs/projects#service-accounts>`_ must have the IAM permission "pubsub.topics.publish". This command checks to see if the destination Cloud Pub/Sub topic grants the service agent this permission. If not, the create command attempts to grant it. A bucket can have up to 100 total notification configurations and up to 10 notification configurations set to trigger for a specific event. <B>CREATE EXAMPLES</B> Begin sending notifications of all changes to the bucket example-bucket to the Cloud Pub/Sub topic projects/default-project/topics/example-bucket: gsutil notification create -f json gs://example-bucket The same as above, but specifies the destination topic ID 'files-to-process' in the default project: gsutil notification create -f json \ -t files-to-process gs://example-bucket The same as above, but specifies a Cloud Pub/Sub topic belonging to the specific cloud project 'example-project': gsutil notification create -f json \ -t projects/example-project/topics/files-to-process gs://example-bucket Create a notification config that only sends an event when a new object has been created: gsutil notification create -f json -e OBJECT_FINALIZE gs://example-bucket Create a topic and notification config that only sends an event when an object beginning with "photos/" is affected: gsutil notification create -p photos/ gs://example-bucket List all of the notificationConfigs in bucket example-bucket: gsutil notification list gs://example-bucket Delete all notitificationConfigs for bucket example-bucket: gsutil notification delete gs://example-bucket Delete one specific notificationConfig for bucket example-bucket: gsutil notification delete \ projects/_/buckets/example-bucket/notificationConfigs/1 <B>OPTIONS</B> The create sub-command has the following options -e Specify an event type filter for this notification config. Cloud Storage only sends notifications of this type. You may specify this parameter multiple times to allow multiple event types. If not specified, Cloud Storage sends notifications for all event types. The valid types are: OBJECT_FINALIZE - An object has been created. OBJECT_METADATA_UPDATE - The metadata of an object has changed. OBJECT_DELETE - An object has been permanently deleted. OBJECT_ARCHIVE - A live version of an object has become a noncurrent version. -f Specifies the payload format of notification messages. Must be either "json" for a payload matches the object metadata for the JSON API, or "none" to specify no payload at all. In either case, notification details are available in the message attributes. -m Specifies a key:value attribute that is appended to the set of attributes sent to Cloud Pub/Sub for all events associated with this notification config. You may specify this parameter multiple times to set multiple attributes. -p Specifies a prefix path filter for this notification config. Cloud Storage only sends notifications for objects in this bucket whose names begin with the specified prefix. -s Skips creation and permission assignment of the Cloud Pub/Sub topic. This is useful if the caller does not have permission to access the topic in question, or if the topic already exists and has the appropriate publish permission assigned. -t The Cloud Pub/Sub topic to which notifications should be sent. If not specified, this command chooses a topic whose project is your default project and whose ID is the same as the Cloud Storage bucket name. <B>NEXT STEPS</B> Once the create command has succeeded, Cloud Storage publishes a message to the specified Cloud Pub/Sub topic when eligible changes occur. In order to receive these messages, you must create a Pub/Sub subscription for your Pub/Sub topic. To learn more about creating Pub/Sub subscriptions, see `the Pub/Sub Subscriber Overview <https://cloud.google.com/pubsub/docs/subscriber>`_. You can create a simple Pub/Sub subscription using the ``gcloud`` command-line tool. For example, to create a new subscription on the topic "myNewTopic" and attempt to pull messages from it, you could run: gcloud beta pubsub subscriptions create --topic myNewTopic testSubscription gcloud beta pubsub subscriptions pull --auto-ack testSubscription a� <B>WATCHBUCKET</B> The watchbucket sub-command can be used to watch a bucket for object changes. A service account must be used when running this command. The app_url parameter must be an HTTPS URL to an application that will be notified of changes to any object in the bucket. The optional id parameter can be used to assign a unique identifier to the created notification channel. If not provided, a random UUID string is generated. The optional token parameter can be used to validate notifications events. To do this, set this custom token and store it to later verify that notification events contain the client token you expect. <B>WATCHBUCKET EXAMPLES</B> Watch the bucket example-bucket for changes and send notifications to an application server running at example.com: gsutil notification watchbucket https://example.com/notify \ gs://example-bucket Assign identifier my-channel-id to the created notification channel: gsutil notification watchbucket -i my-channel-id \ https://example.com/notify gs://example-bucket Set a custom client token that is included with each notification event: gsutil notification watchbucket -t my-client-token \ https://example.com/notify gs://example-bucket a� <B>STOPCHANNEL</B> The stopchannel sub-command can be used to stop sending change events to a notification channel. The channel_id and resource_id parameters should match the values from the response of a bucket watch request. <B>STOPCHANNEL EXAMPLES</B> Stop the notification event channel with channel identifier channel1 and resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8: gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8 a� You can use the ``notification`` command to configure `Pub/Sub notifications for Cloud Storage <https://cloud.google.com/storage/docs/pubsub-notifications>`_ and `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_ channels. <B>CLOUD PUB/SUB</B> The "create", "list", and "delete" sub-commands deal with configuring Cloud Storage integration with Google Cloud Pub/Sub. aP <B>OBJECT CHANGE NOTIFICATIONS</B> Object change notification is a separate, older feature within Cloud Storage for generating notifications. This feature sends HTTPS messages to a client application that you've set up separately. This feature is generally not recommended, because Pub/Sub notifications are cheaper, easier to use, and more flexible. For more information, see `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_. The "watchbucket" and "stopchannel" sub-commands enable and disable Object change notifications. aY <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B> gsutil supports `parallel composite uploads <https://cloud.google.com/storage/docs/uploads-downloads#parallel-composite-uploads>`_. If enabled, an upload can result in multiple temporary component objects being uploaded before the actual intended object is created. Any subscriber to notifications for this bucket then sees a notification for each of these components being created and deleted. If this is a concern for you, note that parallel composite uploads can be disabled by setting "parallel_composite_upload_threshold = 0" in your .boto config file. Alternately, your subscriber code can filter out gsutil's parallel composite uploads by ignoring any notification about objects whose names contain (but do not start with) the following string: "{composite_namespace}". )Zcomposite_namespacea_ Watch bucket attempt failed: {watch_error} You attempted to watch a bucket with an application URL of: {watch_url} which is not authorized for your project. Please ensure that you are using Service Account authentication and that the Service Account's project is authorized for the application URL. Notification endpoint URLs must also be whitelisted in your Cloud Console project. To do that, the domain must also be verified using Google Webmaster Tools. For instructions, please see `Notification Authorization <https://cloud.google.com/storage/docs/object-change-notification#_Authorization>`_. �NONE� JSON_API_V1)�none�jsonc @ s� e Zd ZdZdd� ZdZejdg d�ede dd d d e j ge j e � � e �� gg e �� ge �� ge � � e �d �gd �d � Zejdg d �ddeeeeeed�d�Zeeg d�ede j j d�ede j j d�ed�ed�ed�ed�d�d�eg d �i d�eg d �i d d �d �i d�Z d!d"� Z!d#d$� Z"d%d&� Z#d'd(� Z$d)d*� Z%d9d+d,�Z&d-d.� Z'd/d0� Z(d1d2� Z)d3d4� Z*d5d6� Z+e$e'e)e!e"d�Z,d7d8� Z-dS ):�NotificationCommandz.Implementation of gsutil notification command.c C s t jst�d�t _t jS )Nz]/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/notificationConfigs/(?P<notification>[0-9]+))r �_notification_path_regex�re�compile��self� r$ �*/opt/gsutil/gslib/commands/notification.py�_GetNotificationPathRegex� s �z-NotificationCommand._GetNotificationPathRegexN� notification)�notify� notifyconfig� notificationsZnotif� zi:t:m:t:of:e:p:sF� )� watchbucket� stopchannel�list�delete�create) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r- r. r) � command_helpz$Configure object change notification)r1 r/ r0 r- r. )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�bucketsr* r1 z--custom-attributes)� repeat_typez --event-typesz--object-prefixz--payload-formatz--skip-topic-setupz--topic)�-m�-e�-p�-f�-s�-t)�gcloud_command�flag_map)rD rE r* r0 )rD rE r* r/ z--human-readableT)rM rN �supports_output_translation)r1 r0 r/ c C st | � � d}d}| jr>| jD ] \}}|dkr0|}|dkr |}q |pLtt�� �}| jd }| jd }|�� �d�sxtd��t |�}|� � r�|j dks�td | j ��|� � s�td | j ��| j �d ||� z | jj|j||||j d �}W nB t�y& } z(| j �tjt| �|d �� � W Y d} ~ n d} ~ 0 0 |j} |j} |j}| j �d� | j �d| � | j �d| � | j �d|� dS )z/Creates a watch on a bucket given in self.args.Nz-irL r ���zhttps://z,The application URL must be an https:// URL.�gs�7The %s command can only be used with gs:// bucket URLs.�*URL must name a bucket for the %s command.z.Watching bucket %s with application URL %s ...)�token�provider)Z watch_error� watch_urlz0Successfully created watch notification channel.z Watch channel identifier: %sz%Canonicalized resource identifier: %szClient state token: %s)�CheckArguments�sub_opts�str�uuid�uuid4�args�lower� startswithr r �IsBucket�scheme� command_name�logger�info� gsutil_api� WatchBucket� bucket_namer �warn�)NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE�format�id� resourceIdrT ) r# � identifierZ client_token�o�arV � bucket_arg� bucket_url�channel�e� channel_id� resource_idr$ r$ r% � _WatchBucket� sd ���� � �� z NotificationCommand._WatchBucketc C sF | j d }| j d }| j�d||� | jj||dd� | j�d� dS )Nr r, z3Removing channel %s with resource identifier %s ...rQ �rU z Succesfully removed channel.)r\ rb rc rd � StopChannel)r# rs rt r$ r$ r% � _StopChannel( s � z NotificationCommand._StopChannelc C s� t |�}|�� r|jdks(td| j ��|�� s>td| j ��| jj|jdd�j}| j � d|j� t |�D ]x\}}| j � d|d � | j � d|j � | j � d |j � | j � d |j� | j � d |j� | j � d tt�|jd ��� qjdS )z<Lists active channel watches on a bucket given in self.args.rQ rR rS rv z?Bucket %s has the following active Object Change Notifications:z Notification channel %d:r, z Channel identifier: %sz Resource identifier: %sz Application URL: %sz Created by: %sz Creation time: %si� r )r r_ r` r ra rd � ListChannelsrf �itemsrb rc � enumeraters rt �push_url�subscriber_emailrY r � fromtimestamp�creation_time_ms)r# ro rp �channels�idxrq r$ r$ r% � _ListChannels3 s: ��� ���z!NotificationCommand._ListChannelsc C sl | � � d }d }i }g }d }d}| jr�| jD ]�\}}|dkrH|�|� q,|dkrV|}q,|dkr�d|vrntd��|�dd�\} } | || < q,|dkr�|}q,|d kr�d }q,|d kr,|}q,|tvr�td ��t| }| jd } t| �} | �� r�| � � �std| j | j | f ��| j dk�r td| j ��| j } | j�d| � | jj| dg| j d�}|j}|�sddtd �| f }|�d��s�dtd �|f }| j�d|� d }|�r�| jj|| j d�j}| j�d||� | �||�}tdd�D ]x}z2| jj| ||||�r�|nd || j d�}W �qLW n> t�yF |dk�r@|�r@| j�d � t � d � n� Y n0 �q�d | |j f }| j�d |� dS ) NTrH rJ rG �:zCCustom attributes specified with -m should be of the form key:valuer, rI rK FrL z@Must provide a payload format with -f of either 'json' or 'none'rP z.%s %s requires a GCS bucket name, but got '%s'rQ rR z#Creating notification for bucket %s� projectNumber)�fieldsrU zprojects/%s/topics/%sz projects/z Using Cloud Pub/Sub topic %srv z"Service account for project %d: %sr r+ )� pubsub_topic�payload_format�custom_attributes� event_types�object_name_prefixrU zfRetrying create notification in 10 seconds (new permissions may take up to 10 seconds to take effect.)� z,projects/_/buckets/%s/notificationConfigs/%sz Created notification config %s) rW rX �appendr �split�PAYLOAD_FORMAT_MAPr\ r � IsCloudUrlr_ ra �subcommand_namer` rf rb �debugrd � GetBucketr� r r^ �GetProjectServiceAccount� email_address� _CreateTopic�range�CreateNotificationConfigr rc �time�sleeprj )r# r� r� r� r� r� Zshould_setup_topicrm rn �key�valuero rp rf �bucket_metadata�bucket_project_numberZ just_modified_topic_permissions�service_accountZattempt_numberZcreate_responseZnotification_namer$ r$ r% �_CreateN s� � � �� ���� ���� � � �zNotificationCommand._Createc C s� t | jd�}z |j|d� | j�d|� W n: tyd | j�d|� |j|d� | j�d|� Y n0 |j|d�}tdd| gd�}||j vr�|j � |� |j ||d � d S | j�d |� d S d S )aa Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. )rb )� topic_namezTopic %s already existszCreating topic %sz Created Cloud Pub/Sub topic %szroles/pubsub.publisherzserviceAccount:%s)�role�members)r� �policyTz/GCS already has publish permission to topic %s.FN) r rb ZGetTopicr� r Z CreateTopicrc ZGetTopicIamPolicyr �bindingsr� ZSetTopicIamPolicy)r# r� r� Z pubsub_apir� �bindingr$ r$ r% r� � s( � �z NotificationCommand._CreateTopicc c sD | � � }| jD �].}|�|�}|r�|s<td| j| j|f ��|�d�}|�d�}d}| jj|dd�D ] }|j |krd||fV d} q�qd|s�t d| ��qt |�} | � � s�td | j ��| j dkr�td ��d } | �� r�d } n | �� r�|} | s�td | j ��| �| �jdgd�D ]0} | jj| jjdd�D ]}| jj|fV �q$�q qd S )ay Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) z4%s %s accepts only bucket names, but you provided %s�bucketr' FrQ rv Tz Could not find notification %szJThe %s command must be used on cloud buckets or notification config names.z+The %s command only works on gs:// buckets.Nzgs://*z<The %s command cannot be used on cloud objects, only bucketsrj )� bucket_fields)r& r\ �matchr ra r� �grouprd �ListNotificationConfigsrj r r r� r` � IsProviderr_ �WildcardIterator� IterBuckets� storage_urlrf ) r# �accept_notification_configsZ path_regexZ list_entryr� rf �notification_id�foundr' r� �path�blrr$ r$ r% � _EnumerateNotificationsFromArgs� sb �� � �� �� � � z3NotificationCommand._EnumerateNotificationsFromArgsc C sZ | � � | jr4dt| j�v rV| jD ]}| �|� q"n"| jdd�D ]\}}| �||� q@dS )Nz-oF)r� r )rW rX �dictr\ r� r� �_PrintNotificationDetails�r# rf r' r$ r$ r% �_List s �zNotificationCommand._Listc C s� t dj||j|jtd�d � d�� |jrTt d� |jjD ]}t d|j|jf � q:g }|j rt|� dd� |j � � |j r�|� d|j � |r�t d � |D ] }t |� q�| j �d � d S ) Nz\projects/_/buckets/{bucket}/notificationConfigs/{notification} Cloud Pub/Sub topic: {topic}z//pubsub.googleapis.com/)r� r' �topicz Custom attributes:z %s: %sz Event Types: %sz, z Object name prefix: '%s'z Filters:� )�printri rj r� �lenr� �additionalPropertiesr� r� r� r� �joinr� rb rc )r# r� r' �attr�filters�liner$ r$ r% r� ) s. � �� z-NotificationCommand._PrintNotificationDetailsc C s$ | � � D ]\}}| �||j� qdS )Nr )r� �_DeleteNotificationrj r� r$ r$ r% �_Delete@ s zNotificationCommand._Deletec C s | j j||dd� dS )NrQ )r' rU r )rd �DeleteNotificationConfig)r# rf r� r$ r$ r% r� E s �z'NotificationCommand._DeleteNotificationc C sT z2t � | j| jj�\| _| _tj| jd� || �W S t jyN | �� Y n0 d S )N)rX ) �getoptr\ � command_specr6 rX r �LogCommandParams� GetoptError� RaiseInvalidArgumentException)r# �funcr$ r$ r% �_RunSubCommandK s �� z"NotificationCommand._RunSubCommandc C sT | j �d�| _| jtjv r<tj| jgd� | �tj| j �S td| j| j f ��dS )z1Command entry point for the notification command.r )� subcommandsz+Invalid subcommand "%s" for the %s command.N) r\ �popr� r � SUBCOMMANDSr r� r� r ra r" r$ r$ r% � RunCommand_ s � �z NotificationCommand.RunCommand)T).�__name__� __module__� __qualname__�__doc__r& r r �CreateCommandSpec� _SYNOPSISr r �JSONr �MakeFreeTextArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeZeroOrMoreCloudURLsArgument� MakeNCloudBucketURLsArgumentr� �HelpSpec�_DETAILED_HELP_TEXT�_create_help_text�_list_help_text�_delete_help_text�_watchbucket_help_text�_stopchannel_help_text� help_specr r r �RepeatFlagType�DICT�LIST�gcloud_storage_mapru rx r� r� r� r� r� r� r� r� r� r� r� r$ r$ r$ r% r y s� �����%���������-�15 k. 6 �r )Br� � __future__r r r r r� r r� rZ r �gslibr �gslib.cloud_apir r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.project_idr Zgslib.pubsub_apir �gslib.storage_urlr Z4gslib.third_party.pubsub_apitools.pubsub_v1_messagesr Z gslib.utilsr r �gslib.utils.shim_utilr r �_LIST_SYNOPSIS�_DELETE_SYNOPSIS�_CREATE_SYNOPSISZ_WATCHBUCKET_SYNOPSISZ_STOPCHANNEL_SYNOPSIS�lstripr� �_LIST_DESCRIPTION�_DELETE_DESCRIPTION�_CREATE_DESCRIPTIONZ_WATCHBUCKET_DESCRIPTIONZ_STOPCHANNEL_DESCRIPTIONri � PARALLEL_UPLOAD_TEMP_NAMESPACE� _DESCRIPTIONrh r� r� r� r� r� r� r� r r$ r$ r$ r% �<module> s� ������ x" � � � �����' ������ __pycache__/lifecycle.cpython-39.pyc 0000644 00000014576 15025166704 0013367 0 ustar 00 a +(Wg�" � @ s0 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dZ dZ e e �!d� d Z"dZ#dZ$de# e$ d Z%ee"e%�Z&ee e#�Z'ee e$�Z(G dd� de �Z)dS ) zBImplementation of lifecycle configuration command for GCS buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�LifecycleTranslationz+ gsutil lifecycle get gs://<bucket_name> zA gsutil lifecycle set <config-json-file> gs://<bucket_name>... � aP <B>GET</B> Gets the lifecycle management configuration for a given bucket. You can get the lifecycle management configuration for only one bucket at a time. To update the configuration, you can redirect the output of the ``get`` command into a file, edit the file, and then set it on the bucket using the ``set`` sub-command. z� <B>SET</B> Sets the lifecycle management configuration on one or more buckets. The ``config-json-file`` specified on the command line should be a path to a local file containing the lifecycle configuration JSON document. aG You can use the ``lifecycle`` command to get or set lifecycle management policies for a given bucket. This command is supported for buckets only, not objects. For more information, see `Object Lifecycle Management <https://cloud.google.com/storage/docs/lifecycle>`_. The ``lifecycle`` command has two sub-commands: a� <B>EXAMPLES</B> The following lifecycle management configuration JSON document specifies that all objects in this bucket that are more than 365 days old are deleted automatically: { "rule": [ { "action": {"type": "Delete"}, "condition": {"age": 365} } ] } The following empty lifecycle management configuration JSON document removes all lifecycle configuration for a bucket: {} c s� e Zd ZdZejddgededdddej ej gej e � d�e � � ge �d�gd �d � Zejdd d gd deeed�d�Z� fdd�Zdd� Zdd� Zdd� Z� ZS )�LifecycleCommandz+Implementation of gsutil lifecycle command.� lifecycleZlifecycleconfig� � TF� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZ getlifecycleZ setlifecycle� command_helpz/Get or set lifecycle configuration for a bucket)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� | j d dkrZtdtdddd�| j d �g| j dd � i d �ii d �}| j d d� | _ n td tg d �i d �ii d �}t� �|�S ) Nr r �storage�buckets�updatez--lifecycle-file={}r r )�gcloud_command�flag_mapr )r, r- �describezo--format=gsutiljson[key=lifecycle_config,empty=' has no lifecycle configuration.',empty_prefix_key=storage_url])�argsr �format�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� �'/opt/gsutil/gslib/commands/lifecycle.pyr5 � s6 � ��� ��� �z(LifecycleCommand.get_gcloud_storage_argsc C s� | j d }| j dd� }t|�s.td| j ��t|d�}|�� }|�� d}|D ]�}| j|dgd�}|D ]l}|j} d }| j � d |� | j d kr�| j j || | j d � qht�|�} tj| d �} | j j| j| | j dgd� qhqP|s�ttt|� ��dS )z?Sets lifecycle configuration for a Google Cloud Storage bucket.r r Nz,"%s" command spanning providers not allowed.�rFr �� bucket_fieldsTz(Setting lifecycle configuration on %s...�s3��provider)r �id)rA �fields)r2 r r � command_name�open�read�close�GetBucketUrlIterFromArg� storage_url�logger�info�scheme� gsutil_api�XmlPassThroughSetLifecycler �JsonLifecycleToMessage�apitools_messages�Bucket� PatchBucket� bucket_namer �list) r6 Z lifecycle_arg�url_argsZlifecycle_fileZ lifecycle_txt� some_matched�url_str� bucket_iter�blr�urlr �bucket_metadatar: r: r; �_SetLifecycleConfig� sB � � � � z$LifecycleCommand._SetLifecycleConfigc C sx | j | jd dgd�\}}|jdkr@tj�| jj||jd�� n4|jrd|jj rdtj�t � |j�� ntj�d| � dS )z?Gets lifecycle configuration for a Google Cloud Storage bucket.r r r= r? r@ z#%s has no lifecycle configuration. ) �GetSingleBucketUrlFromArgr2 rL �sys�stdout�writerM �XmlPassThroughGetLifecycler �ruler �JsonLifecycleFromMessage)r6 � bucket_urlr[ r: r: r; �_GetLifecycleConfig� s" � ����z$LifecycleCommand._GetLifecycleConfigc C s^ | j �d�}|dkr*tj|gd� | �� S |dkrHtj|gd� | �� S td|| jf ��dS )z.Command entry point for the lifecycle command.r r )� subcommandsr z+Invalid subcommand "%s" for the %s command.N)r2 �popr �LogCommandParamsre r\ r rD )r6 � subcommandr: r: r; � RunCommand� s �zLifecycleCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSON�XMLr �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr5 r\ re rj � __classcell__r: r: r8 r; r c sJ �� ����� &&r )*rn � __future__r r r r r^ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr rP �gslib.utils.constantsr �gslib.utils.shim_utilr � gslib.utils.translation_helperr � _GET_SYNOPSIS� _SET_SYNOPSIS�lstriprp �_GET_DESCRIPTION�_SET_DESCRIPTION� _DESCRIPTIONrx ry rz r r: r: r: r; �<module> sB ��� __pycache__/rsync.cpython-39.pyc 0000644 00000153561 15025166704 0012564 0 ustar 00 a +(Wg> � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ dd lmZ ddlZdd lmZ dd l m Z dd l m Z dd l m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl0m2Z2 ddl3m4Z4 ddl3m5Z5 ddl3m6Z6 ddl7m8Z9 dd l:m;Z; dd l:m<Z< dd l:m=Z= dd l>m?Z? dd l@mAZA dd!lBmCZC dd"lBmDZD dd#lBmEZE dd$lBmFZF dd%lGmHZH dd&lGmIZI dd'lGmJZJ dd(lGmKZK dd)lLmMZM dd*lLmNZN dd+lLmOZO dd,lPmQZQ dd-lPmRZR dd.lPmSZS dd/lPmTZT dd0lPmUZU dd1lPmVZV dd2lPmWZW dd3lPmXZX dd4lPmYZY dd5lPmZZZ dd6lPm[Z[ dd7lPm\Z\ dd8lPm]Z] dd9lPm^Z^ dd:lPm_Z_ dd;lPm`Z` dd<lPmaZa dd=lPmbZb dd>lPmcZc dd?lPmdZd dd@lemfZf ddAlemgZg ddBlhmiZi ddClhmjZj ddDlkmlZl ddElmmnZn ddFlompZp ddGlomqZq ddHlomrZr ddIlsmtZt eju�r�evZwdJZxdKex dL ZydMZzdNZ{dOZ|g Z}dPdQ� Z~dRdS� ZdTdU� Z�dVdW� Z�dtdXdY�Z�dZd[� Z�d\d]� Z�d^d_� Z�d`da� Z�dbdc� Z�ddde� Z�G dfdg� dge��Z�G dhdi� die��Z�G djdk� dke��Z�dudldm�Z�dndo� Z�dpdq� Z�G drds� dse �Z�dS )vz*Implementation of Unix-like rsync command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�islice)�urllib)�config)�BucketListingObject)�NotFoundException)�ServiceException)�Command)�DummyArgChecker)�'ShimTranslatePredefinedAclSubOptForCopy)�CommandArgument)� ApiSelector)�CommandException)�LogPerformanceSummaryParams)�PluralityCheckableIterator)�SeekAheadResult)�GetCaughtSignals)�RegisterSignalHandler)�GenerationFromUrlAndString)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� constants)� copy_helper)�parallelism_framework_util)�UsingCrcmodExtension)�GetCloudApiInstance)�CreateCopyHelperOpts)� GetSourceFieldsNeededForCopy)�GZIP_ALL_FILES)�SkipUnsupportedObjectError)�%CalculateB64EncodedCrc32cFromContents)�"CalculateB64EncodedMd5FromContents)�SLOW_CRCMOD_RSYNC_WARNING)�SLOW_CRCMOD_WARNING)�CreateCustomMetadata)� GetValueFromObjectCustomMetadata)�ObjectIsGzipEncoded)� ATIME_ATTR)�ConvertDatetimeToPOSIX)�ConvertModeToBase8)�+DeserializeFileAttributesFromObjectMetadata)�GID_ATTR)�InitializePreservePosixData)� MODE_ATTR)� MTIME_ATTR)�NA_ID)�NA_MODE)�NA_TIME)�NeedsPOSIXAttributeUpdate)�ParseAndSetPOSIXAttributes)�POSIXAttributes)�'SerializeFileAttributesToObjectMetadata)�UID_ATTR)� ValidateFilePermissionAccess)�WarnFutureTimestamp)�WarnInvalidValue)�WarnNegativeAttribute)� DiffAction)�RsyncDiffToApply)�GcloudStorageFlag)�GcloudStorageMap)� IS_WINDOWS)�CopyCustomMetadata)�CalculateThroughput)�SECONDS_PER_DAY)�TEN_MIB)�CreateWildcardIteratorz, gsutil rsync [OPTION]... src_url dst_url z <B>SYNOPSIS</B> a S <B>DESCRIPTION</B> The gsutil rsync command makes the contents under dst_url the same as the contents under src_url, by copying any missing files/objects (or those whose data has changed), and (if the -d option is specified) deleting any extra files/objects. src_url must specify a directory, bucket, or bucket subdirectory. For example, to sync the contents of the local directory "data" to the bucket gs://mybucket/data, you could do: gsutil rsync data gs://mybucket/data To recurse into directories use the -r option: gsutil rsync -r data gs://mybucket/data If you have a large number of objects to synchronize you might want to use the gsutil -m option (see "gsutil help options"), to perform parallel (multi-threaded/multi-processing) synchronization: gsutil -m rsync -r data gs://mybucket/data The -m option typically will provide a large performance boost if either the source or destination (or both) is a cloud URL. If both source and destination are file URLs the -m option will typically thrash the disk and slow synchronization down. Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways that can be surprising. Also, attempting to copy files whose names contain wildcard characters can result in problems. For more details about these issues see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. Note 2: If you are synchronizing a large amount of data between clouds you might consider setting up a `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_ account and running gsutil there. Since cross-provider gsutil data transfers flow through the machine where gsutil is running, doing this can make your transfer run significantly faster than running gsutil on your local workstation. Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a `flat namespace <https://cloud.google.com/storage/docs/folders>`_. <B>Using -d Option (with caution!) to mirror source and destination.</B> The rsync -d option is very useful and commonly used, because it provides a means of making the contents of a destination bucket or directory match those of a source bucket or directory. This is done by copying all data from the source to the destination and deleting all other data in the destination that is not in the source. Please exercise caution when you use this option: It's possible to delete large amounts of data accidentally if, for example, you erroneously reverse source and destination. To make the local directory my-data the same as the contents of gs://mybucket/data and delete objects in the local directory that are not in gs://mybucket/data: gsutil rsync -d -r gs://mybucket/data my-data To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete objects in gs://mybucket2 that are not in gs://mybucket1: gsutil rsync -d -r gs://mybucket1 gs://mybucket2 You can also mirror data across local directories. This example will copy all objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1: gsutil rsync -d -r dir1 dir2 To mirror your content across clouds: gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket Change detection works if the other Cloud provider is using md5 or CRC32. AWS multipart upload has an incompatible checksum. As mentioned above, using -d can be dangerous because of how quickly data can be deleted. For example, if you meant to synchronize a local directory from a bucket in the cloud but instead run the command: gsutil -m rsync -r -d ./your-dir gs://your-bucket and your-dir is currently empty, you will quickly delete all of the objects in gs://your-bucket. You can also cause large amounts of data to be lost quickly by specifying a subdirectory of the destination as the source of an rsync. For example, the command: gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket would cause most or all of the objects in gs://your-bucket to be deleted (some objects may survive if there are any with names that sort lower than "data" under gs://your-bucket/data). In addition to paying careful attention to the source and destination you specify with the rsync command, there are two more safety measures you can take when using gsutil rsync -d: 1. Try running the command with the rsync -n option first, to see what it would do without actually performing the operations. For example, if you run the command: gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket it will be immediately evident that running that command without the -n option would cause many objects to be deleted. 2. Enable object versioning in your bucket, which allows you to restore objects if you accidentally delete them. For more details see `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_. <B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B> Running gsutil rsync over a directory containing operating system-specific file types (symbolic links, device files, sockets, named pipes, etc.) can cause various problems. For example, running a command like: gsutil rsync -r ./dir gs://my-bucket will cause gsutil to follow any symbolic links in ./dir, creating objects in my-bucket containing the data from the files to which the symlinks point. This can cause various problems: * If you use gsutil rsync as a simple way to backup a directory to a bucket, restoring from that bucket will result in files where the symlinks used to be. At best this is wasteful of space, and at worst it can result in outdated data or broken applications -- depending on what is consuming the symlinks. * If you use gsutil rsync over directories containing broken symlinks, gsutil rsync will abort (unless you pass the -e option). * gsutil rsync skips symlinks that point to directories. Since gsutil rsync is intended to support data operations (like moving a data set to the cloud for computational processing) and it needs to be compatible both in the cloud and across common operating systems, there are no plans for gsutil rsync to support operating system-specific file types like symlinks. We recommend that users do one of the following: * Don't use gsutil rsync over directories containing symlinks or other OS- specific file types. * Use the -e option to exclude symlinks or the -x option to exclude OS-specific file types by name. * Use a tool (such as tar) that preserves symlinks and other OS-specific file types, packaging up directories containing such files before uploading to the cloud. <B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B> While Google Cloud Storage is strongly consistent, some cloud providers only support eventual consistency. You may encounter scenarios where rsync synchronizes using stale listing data when working with these other cloud providers. For example, if you run rsync immediately after uploading an object to an eventually consistent cloud provider, the added object may not yet appear in the provider's listing. Consequently, rsync will miss adding the object to the destination. If this happens you can rerun the rsync operation again later (after the object listing has "caught up"). <B>FAILURE HANDLING</B> The rsync command retries failures when it is useful to do so, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the overall command fails. If the -C option is provided, the command instead skips failing objects and moves on. At the end of the synchronization run, if any failures were not successfully retried, the rsync command reports the count of failures and exits with non-zero status. At this point you can run the rsync command again, and gsutil attempts any remaining needed copy and/or delete operations. For more details about gsutil's retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. <B>CHANGE DETECTION ALGORITHM</B> To determine if a file or object has changed, gsutil rsync first checks whether the file modification time (mtime) of both the source and destination is available. If mtime is available at both source and destination, and the destination mtime is different than the source, or if the source and destination file size differ, gsutil rsync will update the destination. If the source is a cloud bucket and the destination is a local file system, and if mtime is not available for the source, gsutil rsync will use the time created for the cloud object as a substitute for mtime. Otherwise, if mtime is not available for either the source or the destination, gsutil rsync will fall back to using checksums. If the source and destination are both cloud buckets with checksums available, gsutil rsync will use these hashes instead of mtime. However, gsutil rsync will still update mtime at the destination if it is not present. If the source and destination have matching checksums and only the source has an mtime, gsutil rsync will copy the mtime to the destination. If neither mtime nor checksums are available, gsutil rsync will resort to comparing file sizes. Checksums will not be available when comparing composite Google Cloud Storage objects with objects at a cloud provider that does not support CRC32C (which is the only checksum available for composite objects). See 'gsutil help compose' for details about composite objects. <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (i.e., without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata (like Content-Type and Cache-Control). In contrast, when you download data from the cloud it ends up in a file, which has no associated metadata, other than file modification time (mtime). Thus, unless you have some way to hold on to or re-create that metadata, synchronizing a bucket to a directory in the local file system will not retain the metadata other than mtime. Note that by default, the gsutil rsync command does not copy the ACLs of objects being synchronized and instead will use the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the -p option. See the `Options section <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to learn how. <B>LIMITATIONS</B> 1. The gsutil rsync command will only allow non-negative file modification times to be used in its comparisons. This means gsutil rsync will resort to using checksums for any file with a timestamp before 1970-01-01 UTC. 2. The gsutil rsync command considers only the live object version in the source and destination buckets when deciding what to copy / delete. If versioning is enabled in the destination bucket then gsutil rsync's replacing or deleting objects will end up creating versions, but the command doesn't try to make any noncurrent versions match in the source and destination buckets. 3. The gsutil rsync command does not support copying special file types such as sockets, device files, named pipes, or any other non-standard files intended to represent an operating system resource. If you run gsutil rsync on a source directory that includes such files (for example, copying the root directory on Linux that includes /dev ), you should use the -x flag to exclude these files. Otherwise, gsutil rsync may fail or hang. 4. The gsutil rsync command copies changed files in their entirety and does not employ the `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_ to transfer portions of a changed file. This is because Cloud Storage objects are immutable and no facility exists to read partial object checksums or perform partial replacements. <B>OPTIONS</B> -a predef-acl Sets the specified predefined ACL on uploaded objects. See "gsutil help acls" for further details. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Also see the -p option below. -c Causes the rsync command to compute and compare checksums (instead of comparing mtime) for files if the size of source and destination match. This option increases local disk I/O and run time if either src_url or dst_url are on the local file system. -C If an error occurs, continue to attempt to copy the remaining files. If errors occurred, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rsync...". NOTE: -C only applies to the actual copying operation. If an error occurs while iterating over the files in the local directory (e.g., invalid Unicode file name) gsutil will print an error message and abort. -d Delete extra files under dst_url not found under src_url. By default extra files are not deleted. NOTE: this option can delete data quickly if you specify the wrong source/destination combination. See the help section above, "BE CAREFUL WHEN USING -d OPTION!". -e Exclude symlinks. When specified, symbolic links will be ignored. Note that gsutil does not follow directory symlinks, regardless of whether -e is specified. -i Skip copying any files that already exist at the destination, regardless of their modification time. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the -j extension list. This is useful when uploading files with compressible content (such as .js, .css, or .html files) because it saves network bandwidth while also leaving the data uncompressed in Google Cloud Storage. When you specify the -j option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the Content-Type and name of the original files. Note that if you want to use the top-level -m option to parallelize copies along with the -j/-J options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. This compression buffer size can be changed to a higher limit, e.g.: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \ -m rsync -j html,txt /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the -j option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If you use this option and some of the source files don't compress well (e.g., that's often true of binary data), this option may result in longer uploads. -n Causes rsync to run in "dry run" mode, i.e., just outputting what would be copied or deleted without actually doing any copying/deleting. -p Causes ACLs to be preserved when objects are copied. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Thus, if the source and destination differ in size or modification time and you run gsutil rsync -p, the file will be copied and ACL preserved. However, if the source and destination don't differ in size or checksum but have different ACLs, running gsutil rsync -p will have no effect. Note that this option has performance and cost implications when using the XML API, as it requires separate HTTP calls for interacting with ACLs. The performance issue can be mitigated to some degree by using gsutil -m rsync to cause parallel synchronization. Also, this option only works if you have OWNER access to all of the objects that are copied. You can avoid the additional performance and cost of using rsync -p if you want all objects in the destination bucket to end up with the same ACL by setting a default object ACL on that bucket instead of using rsync -p. See 'gsutil help defacl'. -P Causes POSIX attributes to be preserved when objects are copied. With this feature enabled, gsutil rsync will copy fields provided by stat. These are the user ID of the owner, the group ID of the owning group, the mode (permissions) of the file, and the access/modification timestamps of the file. For downloads, these attributes will only be set if the source objects were uploaded with this flag enabled. On Windows, this flag will only set and restore access time and modification time. This is because Windows doesn't have a notion of POSIX uid/gid/mode. -R, -r The -R and -r options are synonymous. Causes directories, buckets, and bucket subdirectories to be synchronized recursively. If you neglect to use this option gsutil will make only the top-level directory in the source and destination URLs match, skipping any sub-directories. -u When a file/object is present in both the source and destination, if mtime is available for both, do not perform the copy if the destination mtime is newer. -U Skip objects with unsupported object types instead of failing. Unsupported object types are Amazon S3 Objects in the GLACIER storage class. -x pattern Causes files/objects matching pattern to be excluded, i.e., any matching files/objects are not copied or deleted. Note that the pattern is a `Python regular expression <https://docs.python.org/3/howto/regex.html>`_, not a wildcard (so, matching any string ending in "abc" would be specified using ".*abc$" rather than "*abc"). Note also that the exclude path is always relative (similar to Unix rsync or tar exclude options). For example, if you run the command: gsutil rsync -x "data.[/\].*\.txt$" dir gs://my-bucket it skips the file dir/data1/a.txt. You can use regex alternation to specify multiple exclusions, for example: gsutil rsync -x ".*\.txt$|.*\.jpg$" dir gs://my-bucket skips all .txt and .jpg files in dir. NOTE: When using the Windows cmd.exe command line interpreter, use ``^`` as an escape character instead of ``\`` and escape the ``|`` character. When using Windows PowerShell, use ``'`` instead of ``"`` and surround the ``|`` character with ``"``. -y pattern Similar to the -x option, but the command will first skip directories/prefixes using the provided pattern and then exclude files/objects using the same pattern. This is usually much faster, but won't work as intended with negative lookahead patterns. For example, if you run the command: gsutil rsync -y "^(?!.*\.txt$).*" dir gs://my-bucket This would first exclude all subdirectories unless they end in .txt before excluding all files except those ending in .txt. Running the same command with the -x option would result in all .txt files being included, regardless of whether they appear in subdirectories that end in .txt. �-i i' c C s t � dS )zDCalled when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.N)�CleanUpTempFiles)� signal_num�cur_stack_frame� rM �#/opt/gsutil/gslib/commands/rsync.py�_HandleSignals s rO c C sz t D ]} | js| �� qt D ]X} tj�| j�r zt�| j� W q tyr } z t � d| j|� W Y d}~q d}~0 0 q dS )a{ Cleans up temp files. This function allows the main (RunCommand) function to clean up at end of operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary because tempfile.NamedTemporaryFile doesn't allow the created file to be re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which doesn't automatically delete temp files. z;Failed to close and delete temp file "%s". Got an error: %sN) � _tmp_files�closed�close�os�path�isfile�name�unlink� Exception�logging�debug)�fileobj�erM rM rN rJ # s �rJ c C sL |j tjks| jsdS t|j�}|�� rHtj� |j �rH| j � d|� dS dS )z5Arg checker that skips symlinks if -e flag specified.Tz Skipping symbolic link %s...F) � diff_actionr? �REMOVE�exclude_symlinksr � src_url_str� IsFileUrlrS rT �islink� object_name�logger�info)�command_instance� diff_to_apply� exp_src_urlrM rM rN �_DiffToApplyArgChecker@ s � ri c C s� t |�} t |�} | �� r�|tks(| �� rr|tkr<| �d|� t| jd��} t| �}W d � q�1 sf0 Y nX|tks�| �� r�|tkr�| �d|� t| jd��} t| �}W d � n1 s�0 Y | �� �r�|tk�r*|tkr�| �d|� t| jd��} t| �}W d � n1 �s 0 Y nV|tk�r�|tk�rJ| �d|� t| jd��} t| �}W d � n1 �sv0 Y ||||fS )a� Computes any file checksums needed by _CompareObjects. Args: logger: logging.logger for outputting log messages. src_url_str: Source URL string. src_size: Source size src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: (src_crc32c, src_md5, dst_crc32c, dst_md5) zComputing CRC32C for %s...�rbNzComputing MD5 for %s...) r ra �_NArG re �openrc r$ r% ) rd r` �src_size� src_crc32c�src_md5� dst_url_str�dst_size� dst_crc32c�dst_md5�src_url�dst_url�fprM rM rN �_ComputeNeededFileChecksumsM s2 ( & * (rw c C s� t | |d�}|\}}}tj|dtjd�}ztt| |||�|� W nN ty� } z6| j� d||f � | j� t � � � d| _ W Y d}~n d}~0 0 |�� dS )ae Worker function for listing files/objects under to be sync'd. Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We sort the listed URLs because we don't want to depend on consistent sort order across file systems and cloud providers. Args: cls: Command instance. args_tuple: (base_url_str, out_file_name, desc), where base_url_str is top-level URL string to list; out_filename is name of file to which sorted output should be written; desc is 'source' or 'destination'. thread_state: gsutil Cloud API instance to use. �� thread_state�w��mode�encodingz3Caught non-retryable exception while listing %s: %s� N)r �iorl r �UTF8� _BatchSort�_FieldedListingIteratorrX rd �errorrZ � traceback� format_exc� non_retryable_listing_failuresrR ) �clsZ args_tuplery � gsutil_api� base_url_strZ out_filename�desc�out_filer\ rM rM rN �_ListUrlRootFunc{ s �� r� c c sB t �| j�D ]0}t j�| j|�}t j�|�r tt|�d�V q dS )a� A generator that yields a BLR for each file in a local directory. We use this function instead of WildcardIterator for listing a local directory without recursion, because the glob.globi implementation called by WildcardIterator skips "dot" files (which we don't want to do when synchronizing to or from a local directory). Args: base_url: URL for the directory over which to iterate. Yields: BucketListingObject for each file in the directory. N)rS �listdirrc rT �joinrU r r )�base_url�filenamerM rM rN �_LocalDirIterator� s r� c c sz t |�}|jdkr"| js"t|�}n�| jr8d|�d� }nd|�d� }ddddd d t g}| jr�|�d t d t d t d t g� | j d ur�|| j | j fnd }t||| j|| j| jd �j|d �}d} |D ]�} | j} t| | d�r�q�| jr�| �� r�tj�| j�r�q�| j �rD| jt|j�d � } | � | j ��r4| dd � } | j � | ��rDq�| d7 } | t dk�rj| j� d|| � t!| �V q�d S )aV Iterator over base_url_str formatting output per _BuildTmpOutputLine. Args: cls: Command instance. gsutil_api: gsutil Cloud API instance to use for bucket listing. base_url_str: The top-level URL string over which to iterate. desc: 'source' or 'destination'. Yields: Output line formatted per _BuildTmpOutputLine. �filez%s/**�/\z%s/*�crc32c�md5HashrV �size� timeCreatedz metadata/%sN)� project_id� exclude_tuple�ignore_symlinksrd )�bucket_listing_fieldsr )�blrr~ zAt %s listing %d...)"r �scheme�recursion_requestedr� �rstripr2 �preserve_posix_attrs�extendr+ r1 r/ r: �exclude_pattern� exclude_dirsrH r� r_ rd � IterObjects� storage_urlr ra rS rT rb rc � url_string�len� startswith�delim�match� _PROGRESS_REPORT_LISTING_COUNTre �_BuildTmpOutputLine) r� r� r� r� r� �iterator�wildcard�fieldsr� �ir� �url� str_to_checkrM rM rN r� � sp ������� � r� c C s� t }t}t}t}t}t }t }t}| j} | �� r�t�| j�\ }} } } }}} }}} t |�}t |�}t |�}|dk rrt }|dk r~t }n�| � � �rn| j j } | j jdu�rDt| j tt �\} } zDt | �}| r�|t kr�td| j� |t t�� �t kr�td| j� W n$ t�y td| j� t }Y n0 t| j | j�}|jj}|j}|j }|j }t | j j �}| j j �p\t}| j j!�pjt}nt"d| j# ��t$| j�| ||||||||g }dd� |D �}d�%|�d S ) a� Builds line to output to temp file for given BucketListingRef. Args: blr: The BucketListingRef. Returns: The output line, formatted as _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp> gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that aren't composite objects. A missing field is populated with '-', or -1 in the case of atime/mtime/time_created. r N�mtimez Got unexpected URL type (%s)c S s g | ]}t �t|���qS rM )�six� ensure_text�str)�.0r� rM rM rN � <listcomp>R � z'_BuildTmpOutputLine.<locals>.<listcomp>� � )&r5 rk r3 r4 r� ra rS �statrc �longr- � IsCloudUrl� root_objectr� �metadatar) r2 r> r� �timerF r< � ValueErrorr= r. r| � permissions�atime�uid�gidr, r� r� r� r r� � _EncodeUrlr� )r� r� r� r� �md5r| r� � time_createdr� r� �_r� Zfound_mZ mtime_str� posix_attrs�attrsrM rM rN r� sr � � � r� c C s t �| �} tjj| dd�S )z�Encodes url_str with quote plus encoding and UTF8 character encoding. We use this for all URL encodings. Args: url_string (unicode): String URL to encode. Returns: (str) A string encoded using urllib's `quote_plus()` method. � ~)�safe)r� � ensure_strr �parse� quote_plus)r� rM rM rN r� V s r� c C s" t j�| �}tjr |�tj�}|S )z�Inverts encoding from `_EncodeUrl()`. Args: enc_url_string (str): String containing UTF-8-decodable characters that were encoded using urllib's `quote_plus()`. Returns: (unicode) A decoded URL. )r r� � unquote_plusr� �PY2�decoder r� )Zenc_url_stringr� rM rM rN � _DecodeUrli s r� c C s� t �ddd�}g }�z"z|tt| |��}|s,q�tjd|jt|�f dtj d�}|� |� |� t � d�|��� |�� |�d� q|�tj|� � W nF ty� } z.|jtjkr�td �t�d ����� W Y d }~n d }~0 0 W |D ]T}z|�� t� |j� W q� t �y2 } z t � d |j|� W Y d }~q�d }~0 0 q�n^|D ]V}z|�� t� |j� W n6 t �y� } z t � d |j|� W Y d }~n d }~0 0 �q<0 d S ) a� Sorts input lines from in_iter and outputs to out_file. Sorts in batches as input arrives, so input file does not need to be loaded into memory all at once. Derived from Python Recipe 466302: Sorting big files the Python 2.4 way by Nicolas Lehuen. Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line when we could just sort on the first record (URL); but the sort order is identical either way. Args: in_iter: Input iterator. out_file: Output file. �GSUtil�rsync_buffer_linesi } z%s-%06izw+r{ � r r� z�Synchronization failed because too many open file handles were needed while building synchronization state. Please see the comments about rsync_buffer_lines in your .boto config file for a possible way to address this problem.Nz8Failed to remove rsync chunk file "%s". Got an error: %s) r �getint�sortedr r rl rV r� r r� �append�writer� � text_typer� �flush�seek� writelines�heapq�merge�IOError�errnoZEMFILEr �textwrap�wraprR rS �removerX rY rZ )Zin_iterr� � buffer_sizeZ chunk_filesZ current_chunkZ output_chunkr\ Z chunk_filerM rM rN r� z sR � ��� ��r� c @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dS )� _DiffIteratorz7Iterator yielding sequence of RsyncDiffToApply objects.c C s� || _ |j| _|j| _|j| _| j j| _|| _|| _|j| _|j | _ |j | _ | j� d� t j ddd�}t j ddd�}|j| _|j| _t�|� t�|� |�� |�� t| jj| jdf| jj| jdfg�}d|_d g}|jt|t|t|jj d d � |j�rt d ��t | jd �| _ t | jd �| _ t�| j � t�| j � |�!� �rx|�"� �rx| j�rxt#t| j ��| _$| �%� | j �&d� t#t| j ��| _$t#t| j ��| _'d S )Nz!Building synchronization state...zgsutil-rsync-src-F)�prefix�deletezgsutil-rsync-dst-�source� destinationr r� T)� arg_checker� parallel_operations_override� fail_on_errorz/Caught non-retryable exception - aborting rsync�r)(� command_obj�compute_file_checksums� delete_extrasr� rd � base_src_url� base_dst_urlr� �preserve_posix�skip_old_files�ignore_existingre �tempfile�NamedTemporaryFilerV �sorted_list_src_file_name�sorted_list_dst_file_namerP r� rR �iterr� r� �Applyr� � _RootListingExceptionHandlerr �ParallelOverrideReason�SPEEDr rl �sorted_list_src_file�sorted_list_dst_filer� ra r �sorted_src_urls_it�_ValidateObjectAccessr� �sorted_dst_urls_it)�selfr� r� r� Z temp_src_fileZ temp_dst_fileZ args_iter� shared_attrsrM rM rN �__init__� s� �� ���� �� ��z_DiffIterator.__init__c C sz t �� }| jD ]D}| �|�\ }}}}}}}}}}t||||d�\}} |s|�| � q|rv|D ]} | j�| � q\td��dS )a Validates that the user won't lose access to the files if copied. Iterates over the src file list to check if access will be maintained. If at any point we would orphan a file, a list of errors is compiled and logged with an exception raised to the user. )r� r� r| zPThis sync will orphan file(s), please fix their permissions before trying again.N) � collections�dequer �_ParseTmpFileLiner; r� rd �criticalr ) r �errorsrt r` r� �src_mode�src_uid�src_gid�valid�errrM rM rN r s �� z#_DiffIterator._ValidateObjectAccessc C s\ |� dd�\ }}}}}}}} } } t|�t|�t|�t|�t|�t|�t|�t| �| | �� f S )a� Parses output from _BuildTmpOutputLine. Parses into tuple: (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) where crc32c and/or md5 can be _NA and atime/mtime/time_created can be NA_TIME. Args: line: The line to parse. Returns: Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) N� )�rsplitr� �intr� �strip) r �lineZ encoded_urlr� r� r� r� r| r� r� r� r� rM rM rN r ! s ��z _DiffIterator._ParseTmpFileLinec C s2 t |��� r.|tkr.|tkr.| j�d|� dS dS )z�Warns if given url_str is a cloud URL and is missing both crc32c and md5. Args: url_str: Destination URL string. crc32c: Destination CRC32c. md5: Destination MD5. Returns: True if issued warning. zKFound no hashes to validate %s. Integrity cannot be assured without hashes.TF)r r� rk rd �warn)r �url_strr� r� rM rM rN �_WarnIfMissingCloudHash? s ��z%_DiffIterator._WarnIfMissingCloudHashc C s< |t k} |t k} | jp,t|��� o,t|��� } | jr>d| | fS | jr^| r^| r^||k r^d| | fS | s�| r�| r�||kpx||k| | fS ||kr�d| | fS t| j||||||| | � \}}} } |tkr�| tkr�| j� d||� || k| | fS |tk�r| tk�r| j� d||� || k| | fS | � |||��s2| � || | � d| | fS )ai Returns whether src should replace dst object, and if mtime is present. Uses mtime, size, or whatever checksums are available. Args: src_url_str: Source URL string. src_size: Source size. src_mtime: Source modification time. src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size. dst_mtime: Destination modification time. dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: A 3-tuple indicating if src should replace dst, and if src and dst have mtime. FTzComparing md5 for %s and %sz Comparing crc32c for %s and %s) r5 r� r r� r� r� rw rd rk rZ r )r r` rm � src_mtimern ro rp rq � dst_mtimerr rs � has_src_mtime� has_dst_mtimeZ use_hashesrM rM rN �_CompareObjectsT sP * � � � � �z _DiffIterator._CompareObjectsc ! c s� t | jj�d��}t | jj�d��}d}d }}|du r�| j�� rHd}nt| �t| j��\ }}}}} } } } } }t || | | | d�}t ||d� � dd��}t j | jt|�ddd| jd| jd�j}|du �r| j�� �s| �t| j��\ }}}}}}}}}}t ||d� � dd��}|�r�qH|du �s,||k �rFt|||tj|�V d}q0||k�rt| j�rntd|t � tjd�V d}q0t|��� �r�t|��� �r�| tk�r�|} | �||| | ||||||� \} } } | �r�t|||tj|�V nb| j�r t||| || || || |� \}} | �r>t|||tj |�V n | �r>| �s>t|||tj |�V d}d}q0| j�sTdS |�rptd|t � tjd�V | jD ]:} | �| �\ }}}}}}}}}}td|t � tjd�V �qvdS ) zrIterates over src/dst URLs and produces a RsyncDiffToApply sequence. Yields: The RsyncDiffToApply. r� FNT)r� r� r� r� r| �\�/)rt rh �src_url_names_container�have_multiple_srcs�has_multiple_top_level_srcs� exp_dst_url�have_existing_dest_subdirr� ) r� r� r� r� r� r �IsEmptyr �nextr8 r� �replacer �ConstructDstUrlr r� r r@ r? �COPYr� r^ r� ra r5 r! r� r6 �POSIX_SRC_TO_DST�MTIME_SRC_TO_DST)!r Zbase_src_url_lenZbase_dst_url_lenZout_of_src_itemsr` rp rm Zsrc_time_created� src_atimer r r r rn ro r� Zsrc_url_str_to_checkZdst_url_str_would_copy_torq r� � dst_atimer �dst_mode�dst_uid�dst_gidrr rs Zdst_url_str_to_checkZshould_replacer r Z needs_updater rM rM rN �__iter__� s� � ��� ��� � �� � � �� � � �z_DiffIterator.__iter__N) �__name__� __module__� __qualname__�__doc__r r r r r! r5 rM rM rM rN r� � s V Qr� c @ s e Zd ZdZdd� Zdd� ZdS )�_SeekAheadDiffIteratorzGWraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.c C s || _ d S )N)�cloned_diff_iterator)r r; rM rM rN r s z _SeekAheadDiffIterator.__init__c c sB | j D ]6}|jpd}|jtjks,|jtjkr0d}t|d�V qd S )Nr )� data_bytes)r; � copy_sizer] r? r/ r. r )r rg Z bytes_to_copyrM rM rN r5 s �z _SeekAheadDiffIterator.__iter__N)r6 r7 r8 r9 r r5 rM rM rM rN r: s r: c @ s e Zd ZdZdd� ZdS )�$_AvoidChecksumAndListingDiffIteratorz�Iterator initialized from an existing _DiffIterator. This iterator yields RsyncDiffToApply objects used to estimate the total work that will be performed by the DiffIterator, while avoiding expensive computation. c C s� d| _ |j| _|j| _d| _t�d�| _|j| _|j| _|j | _ |j | _ t |j d�| _ t |jd�| _t�| j � t�| j� tt| j ��| _tt| j��| _d S )NF�dummyr� )r� r� r� r� rY � getLoggerrd r� r� r� r� rl r� r r� r rP r� r r� r r )r Zinitialized_diff_iteratorrM rM rN r 7 s. �� ��z-_AvoidChecksumAndListingDiffIterator.__init__N)r6 r7 r8 r9 r rM rM rM rN r> . s r> c C s� t | |d�}|j}t|�}|j}|jtjkr�| jrB| j� d|� n|| j� d|� |� � r�zt � |j � W q� ty� | j�d|� Y q�0 n2z |j|j|j |j|jd� W n ty� Y n0 �n�|jtjk�r.|j}t|�}| j�rr|� � �r^z2t|j d��} W d � n1 �s0 Y W n: t�y\ } z | j� d|j � � W Y d} ~ n d} ~ 0 0 | j� d ||� �q|�z|d} |�� �r�t||j�} |j|j|j | |j| j d �} t | ��r,| j� d | � ndt � � } |j t!t"�"� �t# k�r�t$d |j%� |�&� �s|�'� �r,|�'� �rd nd} | j�(d| |j%� | j)�r<| j)}nt j j*g d�}t+||| j,d� t � � }||_)t-|| dd� t.j/| j|||| t0| | j1d| j2| j3| j,d� }|du�r�|\} }} } | j4� | j5|7 _5W d � n1 �s�0 Y W n8 t6�y( } z | j� d|| j7� W Y d} ~ n d} ~ 0 0 �nN|jtj8k�rJt|j�}| j�r^| j� d|� n�| j� d|j%� |j }t � � }t9t:|i�|_)|�� �r4t|j�}t||j�}z |j;|j|j ||j|jd� W nf t<�y0 } zL| j�d|� | j� d|j%� t=| t>|j|j|tj|j?�|d� W Y d}~n d}~0 0 nt@|j || j,d� �n2|jtjAk�rnt|j�}| j�rz| j� d|� n�| j� d |j%� t � � }t j j*g d�|_)t+||j)dd� |�� �r|t||j�}|j|j|j ||jd gd �}z |j;|j|j ||j|jd� W nf t<�yj } zL| j�d|� | j� d|j%� t=| t>|j|j|tj|j?�|d� W Y d}~n d}~0 0 ntBd |j ��dS ) zEWorker function for performing the actual copy and remove operations.rx zWould remove %sz Removing %sz%s was already removed)� generation�providerrj NzCould not open %szWould copy %s to %s)rA rB r� a� %s has a compressed content-encoding, so it will be decompressed upon download; future executions of gsutil rsync with this source object will always download it. If you wish to synchronize such an object efficiently, compress the source objects in place before synchronizing them, rather than (for example) using gsutil cp -Z to compress them on-the-fly (which results in compressed content-encoding).r� ZStreamsz Named pipeszlWARNING: %s are not supported by gsutil rsync and will likely fail. Use the -x option to exclude %s by name.)�additionalProperties)r� T)�override)�src_obj_metadata�headers�is_rsync� gzip_encoded� gzip_extsr� z0Skipping item %s with unsupported object type %szWould set mtime for %sz$Copying mtime from src to dst for %s)rB rA z Error while trying to patch: %szkCopying whole file/object for %s instead of patching because you don't have patch permission on the object.z!Would set POSIX attributes for %sz/Copying POSIX attributes from src to dst for %s�aclz Got unexpected DiffAction (%d))Cr rp r �src_posix_attrsr] r? r^ �dryrunrd re ra rS rW rc �FileNotFoundErrorrZ � DeleteObject� bucket_namerA r� r r- r` rl rX r� r �GetObjectMetadata�source_metadata_fieldsr* �apitools_messages�Objectr� r� r� rF r< r� �IsFifo�IsStreamr r� � MetadataValuer9 r� rD r � PerformCopy�_RsyncExceptionHandlerrF rH rI � stats_lock�total_bytes_transferredr# �unsupported_typer/ r( r2 �PatchObjectMetadatar � _RsyncFuncr@ r= r7 r. r )r� rg ry r� rp ru r� r` rt r� r\ rE �src_generation� type_text�custom_metadataZtmp_obj_metadataZ copy_result�bytes_transferredr� � obj_metadataZdst_generationr �dst_obj_metadatarM rM rN r] ^ sj � $ �� �� ���� 2� � � ��� � ��� �� ��� �r] c C s | j �t|�� dS )zDSimple exception handler for exceptions during listing URLs to sync.N)rd r� r� �r� r\ rM rM rN r ! s r c C s4 | j �t|�� | jd7 _| j �dt�� � dS )z9Simple exception handler to allow post-completion status.r~ z* Encountered exception while syncing: %s N)rd r� r� �op_failure_countrZ r� r� rd rM rM rN rX & s �rX c s� e Zd ZdZejdg eddddddejej gej e � d�gd� Z ej dd d gd d ei d �Z� fdd�Zdd� Zdd� Zdd� Z� ZS )� RsyncCommandz'Implementation of gsutil rsync command.�rsync� za:cCdenpPriRuUx:y:j:JTFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments�sync� synchronize� command_helpz.Synchronize content of two buckets/directories)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t | j� ddg}dd� | jD �}d|vr@|dg7 }| j�d� d|v r�| jd d �\}\}}t|��� r�t|��� r�| j�d � t|t d �t d �t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�d�d �}t � � |�S ) N�storagerg c S s g | ] \}}|�qS rM rM )r� �flagr� rM rM rN r� M r� z8RsyncCommand.get_gcloud_storage_args.<locals>.<listcomp>�-ez--no-ignore-symlinksz�By default, gsutil copies file symlinks, but, by default, this command (run via the gcloud storage shim) does not copy any symlinks.�-PF)� should_update_sub_opts_and_argsz�For preserving POSIX with rsync downloads, gsutil aborts if a single download will result in invalid destination POSIX. However, this command (run via the gcloud storage shim) will skip invalid copies and still perform valid copies.z--predefined-aclz--checksums-onlyz--continue-on-errorz&--delete-unmatched-destination-objectsz--ignore-symlinksz --no-clobberz--gzip-in-flight-allz--gzip-in-flightz --dry-runz--preserve-posixz--preserve-aclz --recursivez--skip-unsupportedz --skip-if-dest-has-newer-mtimez --exclude)�-a�-c�-C�-dr �-i�-J�-j�-nr� �-p�-R�-r�-U�-u�-x)�gcloud_command�flag_map) r �sub_optsrd r � ParseSubOptsr r� ra rB rA �super�get_gcloud_storage_args)r r� Z flag_keysr� � source_pathZdestination_path�gcloud_storage_map�� __class__rM rN r� I sN �� ����z$RsyncCommand.get_gcloud_storage_argsc C s2 t j|| j| j|| jd�\}}|s.td| ��|S )aw Sanity checks that URL names an existing container. Args: url_str: URL string to check. treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. Returns: URL for checked string. Raises: CommandException if url_str doesn't name an existing container. )rd z�arg (%s) does not name a directory, bucket, or bucket subdir. If there is an object with the same path, please add a trailing slash to specify the directory.)r �ExpandUrlToSingleBlrr� r� rd r )r r �"treat_nonexistent_object_as_subdirr� Zhave_existing_containerrM rM rN �_InsistContainerv s � ��z RsyncCommand._InsistContainerc C s� | � � d| _t�� | _t� s>| jr2| j�t � n | j�t � | � | j d d�}| � | j d d�}|� � o||� � o||j|jk}t|�� |� � |�� |� � || j|j|jgd� t|� � | j| jd| jd�| _d| _d}t� D ]}t|t� q�| jdd| j j dd �\}}t j || j || d � t!| ||�}t"t#|��} | j�$d � t%�%� } z"| j&t'|t(|t)d| d � W t*� nt*� 0 t%�%� } | | | _+t,| j| j+�| _-t| j-| j+| jd � | j�r�| j�r�dnd} t.d| j| | f ��dS )z*Command entry point for the rsync command.r Fr~ T)� has_file_src� has_cloud_src� has_file_dst� has_cloud_dst�is_daisy_chain�uses_fan�provider_types)rG r� )re rZ N)� process_count� thread_countr� �print_macos_warning)� worker_countzStarting synchronization...)r� r� �seek_ahead_iterator)�avg_throughput�total_elapsed_timerZ �sr� z/%d file%s/object%s could not be copied/removed.)/� _ParseOptsrZ r � CreateLockrY r r� rd r r' r& r� �argsr� r� r ra �parallel_operationsr! �skip_unsupported_objects� preserve_aclr� rQ re r r rO �_GetProcessAndThreadCountr r r �.TriggerReauthForDestinationProviderIfNecessaryr� r� r: r> re r� r� r] rX ri rJ r� rE �total_bytes_per_secondr ) r rt ru r� r rK r� r� Z diff_iteratorr� � start_time�end_time� plural_strrM rM rN � RunCommand� s� � �� � � � � �� �zRsyncCommand.RunCommandc C sF d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _ d| _ d| _ d }d | _ d}d }d }| j �r�| j D �]�\}}|dkr�|}d| _ |dkr�d| _ql|dkr�d| _ql|dkr�d| _ql|dkr�d| _ ql|dkr�d}d d � |�d �D �}ql|d k� r�d}t}ql|d k�rd| _ql|dk�r"d| _ql|dk�r@d| _t�s�t� ql|dk�sT|dk�r\d| _ql|dk�rnd| _ ql|dk�r�d| _ ql|dk�r�d| _ ql|dk�s�|dkrl|dk�r�d| _|�s�td��zt�|�| _W ql tj�y� td| ��Y ql0 ql| j�r|�rtd��|�r"|�r"td��|| _|�p0|| _t|| j| j d�S ) NFr� Tr� r� r� r r� c S s g | ] }|� � �qS rM )r )r� �xrM rM rN r� r� z+RsyncCommand._ParseOpts.<locals>.<listcomp>�,r� r� r� r� r� r� r� r� r� r� z-yz Invalid blank exclude filterzInvalid exclude filter (%s)z:Specifying both the -p and -a options together is invalid.z:Specifying both the -j and -J options together is invalid.)� canned_aclr� r� )r_ �continue_on_errorr� r� r� r� rL r� r� r� r� r� �cannedr� �splitr" rC r0 r� r �re�compiler� rH rI r )r r� rH � gzip_arg_exts� gzip_arg_all�o�arM rM rN r� � s� � � �zRsyncCommand._ParseOpts)r6 r7 r8 r9 r �CreateCommandSpec� _SYNOPSISr �XML�JSONr � MakeNCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� � __classcell__rM rM r� rN rf . s6 �� -\rf )N)N)�r9 � __future__r r r r r r� r� r � itertoolsr rY rS r� r� r� r� r� �sysr� � six.movesr �botor �crcmod�gslib.bucket_listing_refr �gslib.cloud_apir r � gslib.commandr r Zgslib.commands.cpr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �"gslib.plurality_checkable_iteratorr �gslib.seek_ahead_threadr �gslib.sig_handlingr r �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr rR Z gslib.utilsr r r Zgslib.utils.boto_utilr � gslib.utils.cloud_api_helperr Zgslib.utils.copy_helperr r! r" r# �gslib.utils.hashing_helperr$ r% r& r' �gslib.utils.metadata_utilr( r) r* �gslib.utils.posix_utilr+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> �gslib.utils.rsync_utilr? r@ �gslib.utils.shim_utilrA rB Zgslib.utils.system_utilrC � gslib.utils.translation_helperrD �gslib.utils.unit_utilrE rF rG Zgslib.wildcard_iteratorrH �PY3r r� r� r� rk Z_OUTPUT_BUFFER_SIZEr� rP rO rJ ri rw r� r� r� r� r� r� r� �objectr� r: r> r] r rX rf rM rM rM rN �<module> s� �� # . 'PP7 k0 D __pycache__/update.cpython-39.pyc 0000644 00000030416 15025166704 0012701 0 ustar 00 a +(Wg�F � @ sN d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ ddlZddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lm Z ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl$m'Z' dZ(de( de'� Z)G dd� de�Z*dS )z5Implementation of update command for updating gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�input)�Command)� ApiSelector)�CommandException)�'CheckAndMaybePromptForAnalyticsEnabling)�RegisterSignalHandler)� system_util)�GetConfigFilePaths)� CERTIFICATE_VALIDATION_ENABLED)�RELEASE_NOTES_URL)�CompareVersions)� DisallowUpdateIfDataInGsutilDir)�LookUpGsutilVersion)�GsutilPubTarballz! gsutil update [-f] [-n] [url] z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> NOTE: This command is not available if you're using a gsutil installation from a package manager or the Cloud SDK. When using the Cloud SDK, use ``gcloud components update``. The gsutil update command downloads the latest gsutil release, checks its version, and offers to let you update to it if it differs from the version you're currently running. Once you say "Y" to the prompt of whether to install the update, the gsutil update command locates where the running copy of gsutil is installed, unpacks the new version into an adjacent directory, moves the previous version aside, moves the new version to where the previous version was installed, and removes the moved-aside old version. Because of this, users are cautioned not to store data in the gsutil directory, since that data will be lost when you update gsutil. (Some users change directories into the gsutil directory to run the command. We advise against doing that, for this reason.) Note also that the gsutil update command will refuse to run if it finds user data in the gsutil directory. By default gsutil update will retrieve the new code from %s, but you can optionally specify a URL to use instead. This is primarily used for distributing pre-release versions of the code to a small group of early test users. NOTE: gsutil periodically checks whether a more recent software update is available. By default this check is performed every 30 days; you can change (or disable) this check by editing the software_update_check_period variable in the .boto config file. Note also that gsutil will only check for software updates if stdin, stdout, and stderr are all connected to a TTY, to avoid interfering with cron jobs, streaming transfers, and other cases where gsutil input or output are redirected from/to files or pipes. Software update periodic checks are also disabled by the gsutil -q option (see 'gsutil help options') <B>OPTIONS</B> -f Forces the update command to offer to let you update, even if you have the most current copy already. This can be useful if you have a corrupted local copy. -n Causes update command to run without prompting [Y/n] whether to continue if an update is available. c @ s� e Zd ZdZejddgeddddddejej gej d � Z ej ddgd d e i d �Z d d� Zg d�Zdd� Zdd� Zdd� Zdd� ZdS )� UpdateCommandz(Implementation of gsutil update command.�update�refreshr � �fnTF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api� command_helpz#Update to the latest gsutil release)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s� t js t jrdS t�� }t�tj�j|kr.dS t � }d� |�}| � |||� g }|D ]0}t t� t�|�tj ��} |�d| |f � qTtd� t�d��d||| jd� |�f dd��dS ) a� Explains what to do if sudo needed to update gsutil software. Happens if gsutil was previously installed by a different user (typically if someone originally installed in a shared file system location, using sudo). Args: tf: Opened TarFile. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to if sudo is needed. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. Raises: CommandException: if errors encountered. N� z sudo chmod %s %s� a Since it was installed by a different user previously, you will need to update using the following commands. You will be prompted for your password, and the install will run as "root". If you're unsure what this means please ask your system administrator for help:z; sudo chmod 0644 %s sudo env BOTO_CONFIG="%s" %s update%sT�� informational)r � IS_CYGWIN� IS_WINDOWS�os�getuid�stat�gslib� GSUTIL_DIR�st_uidr �join�_CleanUpUpdateCommand�oct�S_IMODE�ST_MODE�appendr �textwrap�wrap� gsutil_path) �self�tf�dirs_to_remove�old_cwd�user_idZconfig_file_listZ config_filesZ chmod_cmds� config_file�mode� rF �$/opt/gsutil/gslib/commands/update.py�_ExplainIfSudoNeeded� s2 ����� �z"UpdateCommand._ExplainIfSudoNeeded))Z applications�auto�binZbootZdesktop�devzdocuments and settings�etc�export�homeZkernel�libZlib32�libraryz lost+foundZ mach_kernel�mediaZmnt�net�null�network�opt�private�procz program filesZpython�rootZsbin�scriptsZsrv�sys�system�tmpZusersZusr�varZvolumes�win�win32�windowsZwinntc C s8 |D ].}|sd}|� tj��� | jv rtd| ��qdS )a� Raises Exception if any of dirs is known to be unsafe for gsutil update. This provides a fail-safe check to ensure we don't try to overwrite or delete any important directories. (That shouldn't happen given the way we construct tmp dirs, etc., but since the gsutil update cleanup uses shutil.rmtree() it's prudent to add extra checks.) Args: dirs: List of directories to check. Raises: CommandException: If unsafe directory encountered. rS zKEnsureDirsSafeForUpdate: encountered unsafe directory (%s); aborting updateN)�lstripr0 �sep�lower�unsafe_update_dirsr )r? �dirs�drF rF rG �_EnsureDirsSafeForUpdate� s �z&UpdateCommand._EnsureDirsSafeForUpdatec C st |r |� � | �|� |D ].}zt�|� W q tyF tjsB� Y q0 q|rpzt�|� W n tyn Y n0 dS )a Cleans up temp files etc. from running update command. Args: tf: Opened TarFile, or None if none currently open. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. N) �closerg �shutil�rmtree�OSErrorr r/ r0 �chdir)r? r@ rA rB � directoryrF rF rG r7 � s z#UpdateCommand._CleanUpUpdateCommandc C s� t jrtd��t�� r td��t}|s.td��t� d}d}| jrj| jD ] \}}|dkr\d}|dkrHd}qHg }t� � }|� |� t � � }t � |� |s�| j�d� | j�r4| jd } | �d �s�td ��t| �| ��D ]\\} } | d kr�td ��| j} | �� �r| �� �s|�s0td ��q�| �� �r(| �� s�td��q�nt� } t| j| �} | �rRd}nT| � | �}|� � t t j � dd�d�� }|�!� �"� } W d � n1 �s�0 Y |�s�t j#| k�r�| �$|||� | j�r�td| dd��n tddd��|�sBt%� t&| t j#�\}}|�r0t'd� t(�)d| t j*t+f ��� nt'd| t j*f � | �,|||� |�r\d}nt-d�}|�r||�.� d dk�r�| �$|||� tddd��|�s�| � | �}t/t0j1t0j2� t j �3t j � t j*d ��}tj |d �}tj |d �}|� |� |� |� | �4|� z|j |d � W n@ t5�yR } z&| �$|||� td | ��W Y d}~n d}~0 0 tj6�s`t �7|�D ]�\}}}|D ]H}t � t j � ||�t j8�}t �9|t:j;t:j<B t:j=B t:j>B � t �?|� �qt|D ]N} t � t j � || �t j8�}t �9|t:j@t:jAB t:jBB t:j=B t:j>B � t �?|� �qqft � t j � |dd�t j8�}t �9|t:j@t:j=B t:jAB t:j>B t:jBB � t �?|� t �Ct j*t j � |d �� t �Ct j � |d�t j*� | �$|||� t/t0j1t0jD� | j�d!� d S )"z+Command entry point for the update command.z�The update command is only available for gsutil installed from a tarball. If you installed gsutil via another method, use the same method to update it.aP The update command is disabled for Cloud SDK installs. Please run "gcloud components update" to update it. Note: the Cloud SDK incorporates updates to the underlying tools approximately every 2 weeks, so if you are attempting to update to a recently created release / pre-release of gsutil it may not yet be available via the Cloud SDK.z�Your boto configuration has https_validate_certificates = False. The update command cannot be run this way, for security reasons.Fz-fTz-nz Checking for software update...r z.tar.gzz0The update command only works with tar.gz files.z4Invalid update URL. Must name a single .tar.gz file.zG"update" command does not support "file://" URLs without the -f option.z;Invalid update object URL. Must name a single .tar.gz file.N�gsutil�VERSION�rz You already have %s installed.r, z5You already have the latest gsutil release installed.r+ z�This command will update to the "%s" version of gsutil at %s. NOTE: This a major new version, so it is strongly recommended that you review the release note details at %s before updating to this version, especially if you use gsutil in scripts.z<This command will update to the "%s" version of gsutil at %s�yzProceed? [y/N] zNot running update.z..)�dir)�pathzUpdate failed: %s.�oldzUpdate complete.)Er3 �IS_PACKAGE_INSTALLr r �InvokedViaCloudSdkr r �sub_opts�tempfile�mkdtempr; r0 �getcwdrl �logger�info�args�endswith� enumerate�WildcardIterator� storage_url� IsFileUrl� IsDirectory� IsCloudUrl�IsObjectr r � gsutil_api�_FetchAndOpenGsutilTarball� extractall�openrs r6 �read�stripro r7 r r �printr< r= r4 r rH r rc r �signal�SIGINT�SIG_IGN�normpathrg � Exceptionr/ �walk�O_RDONLY�fchmodr2 �S_IWRITE�S_IRUSR�S_IRGRP�S_IROTHrh �S_IRWXU�S_IXGRP�S_IXOTH�rename�SIG_DFL) r? �https_validate_certificatesZ force_updateZ no_prompt�o�unused_arA Ztmp_dirrB �update_from_url_str�i�resultr� Ztarball_versionr@ Zver_file�_�major�answerZgsutil_bin_parent_dirZold_dirZnew_dir�e�dirnameZsubdirs� filenames�filename�fd�subdirrF rF rG � RunCommand s ��� ���� ,��� ���� � "����� zUpdateCommand.RunCommandc C s2 | j jd|dg| j| jdd� t�d�}d|_|S )N�cpzfile://gsutil.tar.gzT)�skip_update_checkz gsutil.tar.gzr )�command_runner�RunNamedCommand�headers�debug�tarfiler� Z errorlevel)r? r� r@ rF rF rG r� � s � z(UpdateCommand._FetchAndOpenGsutilTarballN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �XML�JSON� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrH rd rg r7 r� r� rF rF rF rG r f s: �� 5, 1r )+r� � __future__r r r r r0 ri r� r2 rZ r� rx r< � six.movesr r3 � gslib.commandr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �gslib.sig_handlingr Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr Zgslib.utils.text_utilr �gslib.utils.update_utilr r r r� r� r rF rF rF rG �<module> sH �.��3 __pycache__/pap.cpython-39.pyc 0000644 00000015034 15025166704 0012176 0 ustar 00 a +(Wg% � @ s$ d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZdZ ee � d� Z dZ dZ de e Z!ee e!�Z"eee �Z#ee e �Z$ej%j&Z&de�'� d d d Z(G dd� de�Z)dS ) z/This module provides the pap command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz= gsutil pap set (enforced|inherited) gs://<bucket_name>... z( gsutil pap get gs://<bucket_name>... � a <B>SET</B> The ``pap set`` command configures public access prevention for Cloud Storage buckets. If you set a bucket to be ``inherited``, it uses public access prevention only if the bucket is subject to the `public access prevention <https://cloud.google.com/storage/docs/org-policy-constraints#public-access-prevention>`_ organization policy constraint. <B>SET EXAMPLES</B> Configure ``redbucket`` and ``bluebucket`` to use public access prevention: gsutil pap set enforced gs://redbucket gs://bluebucket a <B>GET</B> The ``pap get`` command returns public access prevention values for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if ``redbucket`` and ``bluebucket`` are using public access prevention: gsutil pap get gs://redbucket gs://bluebucket z� The ``pap`` command is used to retrieve or configure the `public access prevention <https://cloud.google.com/storage/docs/public-access-prevention>`_ setting of Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. z)--format=value[separator=": "](name.sub("z ","gs://"),z.iamConfiguration.publicAccessPrevention.yesno(zno="inherited"))c @ s� e Zd ZdZejddgededdddej gej e � d�ge dd d gd �e � � gd �d � Z ejddgddeeed �d�Zeedddedgi dd�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%S )&� PapCommandz"Implements the gsutil pap command.ZpapZpublicaccessprevention� � F� �mode�enforced� inherited)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz"Configure public access prevention)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r- r. �updatez--public-access-prevention)r0 r1 )r- r. r3 z --no-public-access-prevention)r r c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r: �!/opt/gsutil/gslib/commands/pap.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z7PapCommand._ValidateBucketListingRefAndReturnBucketNamec C sP |j }| jj|jdg|jd�}|j}|jp,d}t|��d�}t d||f � dS )z7Gets the public access prevention setting for a bucket.�iamConfiguration��fields�providerr �/z%s: %sN) r5 � gsutil_api� GetBucket� bucket_namer6 r= �publicAccessPrevention�str�rstrip�print)r8 r9 � bucket_url�bucket_metadata� iam_config�public_access_prevention�bucketr: r: r; �_GetPublicAccessPrevention� s � z%PapCommand._GetPublicAccessPreventionc C sV |j }t� }||_tj|d�}td|t|��d�f � | jj |j |dg|j d� dS )zMSets the Public Access Prevention setting for a bucket enforced or inherited.)r= z*Setting Public Access Prevention %s for %srA r= r> r ) r5 �IamConfigurationValuerE �apitools_messages�BucketrH rF rG rB � PatchBucketrD r6 )r8 r9 � setting_argrI rK rJ r: r: r; �_SetPublicAccessPrevention� s � �z%PapCommand._SetPublicAccessPreventionc C s | j �d�}|dvr td��d}g }d}|dkr8| j}n$|dkr\| j}| j �d�}|�|� | j�d�tj kr�td� t � d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j�| jj�tj kr�td� t � d | j ����d }|| g|�R � q�q�|�sttt|� ��dS ) z-Handles pap command on Cloud Storage buckets.r r zpap only supports get|setNr r r4 r z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS Buckets.T)�args�popr rN rT �appendrB �GetApiSelectorr �JSON�join�textwrap�wrapr7 �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr5 r6 r r/ ) r8 � subcommand�subcommand_func�subcommand_argsrS � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr: r: r; �_Pap� sP �� ����zPapCommand._Papc C s\ | j d }| jdd� |dks&|dkrLtj| jd� tj|gd� | �� n td| ��d S ) z(Command entry point for the pap command.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rU � ParseSubOptsr �LogCommandParamsri rg r )r8 �action_subcommandr: r: r; � RunCommand� s �zPapCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rY r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_LIST_FORMAT�gcloud_storage_mapr< rN rT rg rn r: r: r: r; r \ sp ����� ��� ����$�( -r )*rr � __future__r r r[ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rP �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprt �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONry r{ rz rQ rO �get_format_flag_caretr} r r: r: r: r; �<module> sL �� ���� __pycache__/rb.cpython-39.pyc 0000644 00000007441 15025166704 0012024 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZde d ZG dd� de �ZdS )z@Implementation of rb command for deleting cloud storage buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�NotEmptyException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�StorageUrlFromString)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMapz( gsutil rb [-f] gs://<bucket_name>... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Delete one or more buckets. Buckets must be empty before you can delete them. Be certain you want to delete a bucket before you do so, as once it is deleted the name becomes available and another user may create a bucket with that name. (But see also "DOMAIN NAMED BUCKETS" under "gsutil help naming" for help carving out parts of the bucket name space.) <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing buckets. If some buckets couldn't be removed, gsutil's exit status will be non-zero even if this flag is set. If no buckets could be removed, the command raises a "no matches" error. c @ s� e Zd ZdZejdg d�ededdddej ej gej e � � gd� Z ejdg d�d d ei d �Zeg d �d ed�id�Zdd� ZdS )� RbCommandz$Implementation of gsutil rb command.�rb)Z deletebucketZ removebucketZ removebuckets�rmdir� �fFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRemove buckets)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�delete�-fz--continue-on-error)�gcloud_command�flag_mapc C sh d| _ | jr*| jD ]\}}|dkrd| _ qd}d}| jD �]}t|�}|�� rVtd��z t| �|�jdgd��}W n d}| j r�Y q8n� Y n0 |D ]�}|j } | j � d| � z| j j | j| jd� W nr t�y } z@d}| j r�W Y d } ~ q�n d | jv �r td | ��n� W Y d } ~ n"d } ~ 0 d}| j �s4� Y n0 d}q�q8|�sZttt| j� ��|�rdd S d S )z'Command entry point for the rb command.Fr* Tz."rb" command requires a provider or bucket URL�id)� bucket_fieldszRemoving %s...)�providerN�VersionedBucketNotEmptyzqBucket is not empty. Note: this is a versioned bucket, so to delete all objects you need to use: gsutil rm -r %sr r )�continue_on_error�sub_opts�argsr �IsObjectr �list�WildcardIterator� IterBuckets� storage_url�logger�info� gsutil_api� DeleteBucket� bucket_name�schemer �reasonr ) �self�o�unused_a� did_some_workZ some_failed�url_str� wildcard_urlZblrs�blr�url�e� rI � /opt/gsutil/gslib/commands/rb.py� RunCommandn sT � �zRbCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_maprK rI rI rI rJ r < s@ ������r N)rO � __future__r r r r �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.storage_urlr �gslib.utils.constantsr �gslib.utils.shim_utilr r rQ rW r rI rI rI rJ �<module> s* �� __pycache__/cors.cpython-39.pyc 0000644 00000015075 15025166704 0012371 0 ustar 00 a +(Wg," � @ sN d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z dZ!dZ"dZ#dZ$e"e!�%d� d Z&dd�'e#e$g� d Z(ee&e(�Z)ee!e#�Z*ee"e$�Z+G d d � d e �Z,dS ) z=Implementation of cors configuration command for GCS buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�CorsTranslation)�REMOVE_CORS_CONFIGz& gsutil cors get gs://<bucket_name> z: gsutil cors set <cors-json-file> gs://<bucket_name>... z� <B>GET</B> Gets the CORS configuration for a single bucket. The output from ``cors get`` can be redirected into a file, edited and then updated using ``cors set``. z� <B>SET</B> Sets the CORS configuration for one or more buckets. The ``cors-json-file`` specified on the command line should be a path to a local file containing a JSON-formatted CORS configuration, such as the example described above. � z a Gets or sets the Cross-Origin Resource Sharing (CORS) configuration on one or more buckets. This command is supported for buckets only, not objects. An example CORS JSON file looks like the following: [ { "origin": ["http://origin1.example.com"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 3600 } ] The above CORS configuration explicitly allows cross-origin GET requests from http://origin1.example.com and may include the Content-Type response header. The preflight request may be cached for 1 hour. Note that requests to the authenticated browser download endpoint ``storage.cloud.google.com`` do not allow CORS requests. For more information about supported endpoints for CORS, see `Cloud Storage CORS support <https://cloud.google.com/storage/docs/cross-origin#server-side-support>`_. The following (empty) CORS JSON file removes any CORS configuration for a bucket: [] The cors command has two sub-commands: z� For more info about CORS generally, see https://www.w3.org/TR/cors/. For more info about CORS in Cloud Storage, see the `CORS concept page <https://cloud.google.com/storage/docs/cross-origin>`_. c @ s� e Zd ZdZejdddgededdddej ej gej e � d�e � � ge �d�gd �d � Zejdg d �d d eeed�d�Zeeg d�i d�eg d�i d�d�i d�Zdd� Zdd� Zdd� Zdd� ZdS ) � CorsCommandz&Implementation of gsutil cors command.�cors�getcors�setcors� � F� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r r z cross-origin� command_helpz7Get or set a CORS configuration for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describeze--format=gsutiljson[key=cors_config,empty=' has no CORS configuration.',empty_prefix_key=storage_url])�gcloud_command�flag_map)r0 r1 �updatez --cors-filec C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r r )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r: �"/opt/gsutil/gslib/commands/cors.py�_CalculateUrlsStartArg� s z"CorsCommand._CalculateUrlsStartArgc C s | j d }| j dd� }t|�s.td| j ��t|d�}|�� }|�� | j�t |d �j �| _ d}|D ]�}| j |dgd�}|D ]t}|j } d }| j�d |� | j d kr�| jj|| | j d � q�t�|�} | s�t} tj| d �} | jj| j| | j dgd� q�qh|�sttt|� ��dS )z9Sets CORS configuration on a Google Cloud Storage bucket.r r Nz,"%s" command spanning providers not allowed.�rF�id�� bucket_fieldsTzSetting CORS on %s...�s3��provider)r )rC �fields)r6 r r � command_name�open�read�close� gsutil_api�GetApiSelectorr �scheme�api�GetBucketUrlIterFromArg� storage_url�logger�info�XmlPassThroughSetCorsr �JsonCorsToMessageEntriesr �apitools_messages�Bucket� PatchBucket� bucket_namer �list) r9 Zcors_arg�url_argsZ cors_fileZcors_txt� some_matched�url_str� bucket_iter�blr�urlr �bucket_metadatar: r: r; �_SetCors� sH � � � � zCorsCommand._SetCorsc C sp | j | jd dgd�\}}|jdkr@tj�| jj||jd�� n,|jr\tj�t � |j�� ntj�d| � dS )z:Gets CORS configuration for a Google Cloud Storage bucket.r r r? rA rB z %s has no CORS configuration. ) �GetSingleBucketUrlFromArgr6 rK �sys�stdout�writerI �XmlPassThroughGetCorsr r �MessageEntriesToJson)r9 � bucket_urlr^ r: r: r; �_GetCors� s � �� �zCorsCommand._GetCorsc C sR | j �d�}|dkr | j}n"|dkr,| j}ntd|| jf ��tj|gd� |� S )z)Command entry point for the cors command.r r r zCInvalid subcommand "%s" for the %s command. See "gsutil help cors".)� subcommands)r6 �poprg r_ r rE r �LogCommandParams)r9 �action_subcommand�funcr: r: r; � RunCommand� s ��zCorsCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �gcloud_storage_mapr< r_ rg rm r: r: r: r; r g sZ � ����� ���*r )-rq � __future__r r r r ra �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr rS �gslib.utils.constantsr �gslib.utils.shim_utilr � gslib.utils.translation_helperr r � _GET_SYNOPSIS� _SET_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�lstriprs �join� _DESCRIPTIONr{ r| r} r r: r: r: r; �<module> sB � �" __pycache__/ubla.cpython-39.pyc 0000644 00000015725 15025166704 0012350 0 ustar 00 a +(Wg4( � @ sV d Z ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z ddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddl m Z dZ dZ e e � d� Z!dZ"dZ#de# e" Z$ee!e$�Z%ee e"�Z&ee e#�Z'ej(j)Z)e)j*Z+de �,� d d d d e �,� d d d Z-G d d � d e �Z.dS )!z+This module provides the command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� InsistOnOrOff)� shim_utilz2 gsutil ubla set (on|off) gs://<bucket_name>... z! gsutil ubla get bucket_url... � a~ <B>SET</B> The ``ubla set`` command enables or disables uniform bucket-level access for Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil ubla set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil ubla set off gs://redbucket gs://bluebucket a <B>GET</B> The ``ubla get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket(s). <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil ubla get gs://redbucket gs://bluebucket z� The ``ubla`` command is used to retrieve or configure the `uniform bucket-level access <https://cloud.google.com/storage/docs/bucket-policy-only>`_ setting of Cloud Storage bucket(s). This command has two sub-commands, ``get`` and ``set``. z--format=multi[terminator="z2"](name:format="value(format('Uniform bucket-levelz! access setting for gs://{}:'))",zD iamConfiguration.uniformBucketLevelAccess.enabled.yesno(no="False")z:format="value[terminator='z '](format(' Enabled: {}'))",zC iamConfiguration.uniformBucketLevelAccess.lockedTime.sub("T", " ")z,:format="value(format(' LockedTime: {}'))")c @ s� e Zd ZdZejddgededdddej gej e � d�ge dd d gd �e � � gd �d � Z ejddgddeeed �d�Zeedddedgi d�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#S )$� UblaCommandz#Implements the gsutil ubla command.�ublaZuniformbucketlevelaccess� � F� �mode�on�off)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz%Configure Uniform bucket-level access)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--raw)�gcloud_command�flag_map)r/ r0 �updatez --uniform-bucket-level-access)r/ r0 r4 z --no-uniform-bucket-level-access)r r c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r; �"/opt/gsutil/gslib/commands/ubla.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z8UblaCommand._ValidateBucketListingRefAndReturnBucketNamec C s� | � |� |j}| jj|jdg|jd�}|j}|j}t|�� d�|j d�}d}|j rb|j |d< d}|r~t d| j f i |��� d S ) z:Gets the Uniform bucket-level access setting for a bucket.�iamConfiguration��fields�provider�/)�bucket�enabledr � locked_timez LockedTime: {locked_time} zGUniform bucket-level access setting for {bucket}: Enabled: {enabled} N)r= r6 � gsutil_api� GetBucket� bucket_namer7 r> �bucketPolicyOnly�str�rstriprD � lockedTime�print�format)r9 r: � bucket_url�bucket_metadata� iam_configZuniform_bucket_level_accessr@ �locked_time_liner; r; r< �_GetUbla� s* � � ��zUblaCommand._GetUblac C s~ | � |� |j}t� }t� |_|dk|j_tj|d�}|dkrBdnd}td|t |�� d�f � | j j |j |dg|jd� d S ) zDSets the Uniform bucket-level access setting for a bucket on or off.r )r> �Enabling� Disablingz(%s Uniform bucket-level access for %s...rB r> r? r )r= r6 �IamConfigurationValue� uniformBucketLevelAccessValuerI rD �apitools_messages�BucketrM rJ rK rF � PatchBucketrH r7 )r9 r: � setting_argrO rQ rP � setting_verbr; r; r< �_SetUbla� s � �zUblaCommand._SetUblac C s� | j �d�}|dvr td��d}g }d}|dkr8| j}n.|dkrf| j}| j �d�}t|d� |�|� d}| j }|s|| �� |D ],}| �|�}|D ]} d }|| g|�R � q�q�|s�tt t |� ��dS ) z/Handles ubla command on a Cloud Storage bucket.r r zubla only supports get|setNr r z-Only on and off values allowed for set optionFT) �args�popr rS r] r �append�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr r1 ) r9 � subcommand�subcommand_func�subcommand_argsr[ � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr; r; r< �_Ubla� s6 � zUblaCommand._Ublac C s� | j jdd�tjkr.td�t�d| j ����| j d }| j dd� |dksT|d krzt j | j d � t j |gd � | �� n td | ��d S )z)Command entry point for the ubla command.r5 )rA r zBThe "%s" command can only be used with the Cloud Storage JSON API.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rF �GetApiSelectorr �JSONr �join�textwrap�wrapr8 r^ � ParseSubOptsr �LogCommandParamsrm rk )r9 �action_subcommandr; r; r< � RunCommand s" ��� �zUblaCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rp r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr= rS r] rk rw r; r; r; r< r c sp ����� ��� ����$�($r )/r{ � __future__r r �getoptrr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rX �gslib.utils.constantsr �gslib.utils.shim_utilr �gslib.utils.text_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripr} �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� rY rV �BucketPolicyOnlyValuerW �get_format_flag_newliner� r r; r; r; r< �<module> sd �� ��������� __pycache__/setmeta.cpython-39.pyc 0000644 00000026375 15025166704 0013072 0 ustar 00 a +(Wgs? � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l m Z dd l mZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl(m*Z* ddl+m,Z, dd l+m-Z- dd l.m/Z/ dd l.m0Z0 dd l.m1Z1 e!j2Z3d Z4d!e4 d" Z5g d#�Z6g d$�Z7d%d&� Z8d+d'd(�Z9G d)d*� d*e�Z:dS ),zDImplementation of setmeta command for setting cloud object metadata.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�AccessDeniedException)�PreconditionException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�storage_v1_messages)�MetadataMessage)� constants)�parallelism_framework_util)�GetCloudApiInstance)�IsCustomMetadataHeader)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�InsistAsciiHeader)�InsistAsciiHeaderValue)�CopyObjectMetadata)�ObjectMetadataFromHeaders)�PreconditionsFromHeadersz6 gsutil setmeta -h [header:value|header] ... url... z <B>SYNOPSIS</B> u� <B>DESCRIPTION</B> The gsutil setmeta command allows you to set or remove the metadata on one or more objects. It takes one or more header arguments followed by one or more URLs, where each header argument is in one of two forms: - If you specify ``header:value``, it sets the provided value for the given header on all applicable objects. - If you specify ``header`` (with no value), it removes the given header from all applicable objects. For example, the following command sets the ``Content-Type`` and ``Cache-Control`` headers while also removing the ``Content-Disposition`` header on the specified objects: gsutil setmeta -h "Content-Type:text/html" \ -h "Cache-Control:public, max-age=3600" \ -h "Content-Disposition" gs://bucket/*.html If you have a large number of objects to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m setmeta -h "Content-Type:text/html" \ -h "Cache-Control:public, max-age=3600" \ -h "Content-Disposition" gs://bucket/*.html You can also use the setmeta command to set custom metadata on an object: gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object Custom metadata is always prefixed in gsutil with ``x-goog-meta-``. This distinguishes it from standard request headers. Other tools that send and receive object metadata by using the request body do not use this prefix. While gsutil supports custom metadata with arbitrary Unicode values, note that when setting metadata using the XML API, which sends metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. The setmeta command reads each object's current generation and metageneration and uses those as preconditions unless they are otherwise specified by top-level arguments. For example, the following command sets the custom metadata ``icecreamflavor:vanilla`` if the current live object has a metageneration of 2: gsutil -h "x-goog-if-metageneration-match:2" setmeta -h "x-goog-meta-icecreamflavor:vanilla" See `Object metadata <https://cloud.google.com/storage/docs/metadata>`_ for more information about object metadata. <B>OPTIONS</B> -h Specifies a header:value to be added, or header to be removed, from each named object. )z cache-controlzcontent-dispositionzcontent-encodingzcontent-languagez content-typez custom-time)�storage�objects�updatec C s | j �|� d| _dS )zDException handler that maintains state about post-completion status.FN)�logger�error�everything_set_okay)�cls�e� r' �%/opt/gsutil/gslib/commands/setmeta.py� _SetMetadataExceptionHandler� s r) c C s | j ||d� d S )N�� thread_state)�SetMetadataFunc)r% �name_expansion_resultr+ r' r' r( �_SetMetadataFuncWrapper� s r. c s� e Zd ZdZejddgedejdddde j e j ge j e � � gd� Zejddgdd ei d �Zeei d �Z� fd d �Zdd� Zeeddd�ddd��Zdd� Zdd� Z� Z S )�SetMetaCommandz)Implementation of gsutil setmeta command.�setmetaZ setheader� zh:rRF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz(Set metadata on already uploaded objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text��gcloud_command�flag_mapc s� g }| j D ]\}}|dv r dg} q&q | �| �� �\}}g | _ | jdd� |D �dd�}| j|dd�}t| | | }t|i d�} t� �| �S ) N)z-rz-Rz --recursivec S s i | ] }|d �qS )Nr' )�.0Z clear_keyr' r' r( � <dictcomp>� � z:SetMetaCommand.get_gcloud_storage_args.<locals>.<dictcomp>T)�unsetFrD )�sub_opts�_ParseMetadataHeaders� _GetHeaderStringsFromSubOpts�_translate_headers� _GCLOUD_OBJECTS_UPDATE_COMMANDr �super�get_gcloud_storage_args) �selfZrecursive_flag�o�_Z clear_setZset_dict� clear_flags� set_flags�command�gcloud_storage_map�� __class__r' r( rQ � s* �� ��z&SetMetaCommand.get_gcloud_storage_argsc C sN | � | �� �\}}|| _|D ]}d| j|< q | js:td��t| j�dkr~| js~t| jd �}|�� rl|� � s~td| jd ��d| _ t | j �| _ t| j| j| j| j| j| j| j| jg d�d� }t| j| j| �� | j| j| j| jd �}z| jt|td|d � W n< t�y8 } z"|j d k�r"| � � � W Y d }~n d }~0 0 | j �sJtd ��dS )z,Command entry point for the setmeta command.� z|gsutil setmeta requires one or more headers to be provided with the -h flag. See "gsutil help setmeta" for more information.r1 r z URL (%s) must name an objectT)� generation�metadata�metageneration)� all_versions�continue_on_error�bucket_listing_fields)r_ � project_id)� fail_on_error�seek_ahead_iteratori� Nz+Metadata for some objects could not be set.) rL rM �metadata_changer �len�args�recursion_requestedr � IsCloudUrl�IsObjectr$ r �headers� preconditionsr � command_name�debugr" � gsutil_apir_ �parallel_operationsr �GetSeekAheadGsutilApirb �Applyr. r) r �status�_WarnServiceAccounts)rR �metadata_minus� metadata_plus�header�url�name_expansion_iteratorrd r& r' r' r( � RunCommand� sd � � � � � zSetMetaCommand.RunCommand� )�tries� timeout_secsNc C s� t | |d�}|j}| j�d|� t�tj|j�}t | j j | j j d�}|j du rV|j |_ |j du rh|j|_ t| j�}|j|jd�}|tjkr�n(|tjkr�t||dd� |}d|_ d|_|j|j|j||j ||jdgd � t|jt t � � d �� dS ) z�Sets metadata on an object. Args: name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. r* zSetting metadata on %s...)� gen_match�meta_gen_matchN)�providerT)�override�id)r\ rl r� �fields)� message_time) r �expanded_storage_urlr" �infor � JsonToMessage�apitools_messages�Object�expanded_resultr rl r~ r r\ r^ r re �GetApiSelector�schemer �XML�JSONr �PatchObjectMetadata� bucket_name� object_name�_PutToQueueWithTimeout� status_queuer �time) rR r- r+ ro � exp_src_url�cloud_obj_metadatarl �patch_obj_metadata�apir' r' r( r, sD �� � �z SetMetaCommand.SetMetadataFuncc C sL | j s g S g }| j D ]2\}}|dkrd|v s4d|v r<td��|�|� q|S )a: Gets header values from after the "setmeta" part of the command. Example: $ gsutil -h not:parsed setmeta is:parsed gs://bucket/object -> ["is:parsed"] Returns: List[str]: Headers without the "-h" but not yet split on colons. Raises: CommandException Found canned ACL. z-hz x-goog-aclz x-amz-aclzWgsutil setmeta no longer allows canned ACLs. Use gsutil acl set ... to set canned ACLs.)rK r �append)rR rk rS �ar' r' r( rM @ s � z+SetMetaCommand._GetHeaderStringsFromSubOptsc C sV t � }t � }i }i }d}d}d}d} |D ]�} | �d�} | \} } }t| � | �� }t|�}|s||tvr|td| ttt��f ��|r�|r�||| < |d7 }q�t | |� t |�}|||< |d7 }q(|r�|� | � | d7 } q(|� |� |d7 }q(|t |�k�s2|t |�k�s2|t |�k�s2| t |�k�s2|� t |�� ���r:td��|�|� |�|� ||fS )z�Validates and parses metadata changes from the headers argument. Args: headers: Header dict to validate and parse. Returns: (metadata_plus, metadata_minus): Tuple of header sets to add and remove. r �:zhInvalid or disallowed header (%s). Only these fields (plus x-goog-meta-* fields) can be set or unset: %sr1 z%Each header must appear at most once.)�set� partitionr �lowerr �SETTABLE_FIELDSr �sorted�listr �str�addrf � intersection�keysr! )rR rk ru Zcust_metadata_minusrv Zcust_metadata_plusZnum_metadata_plus_elemsZ num_cust_metadata_plus_elemsZnum_metadata_minus_elemsZ num_cust_metadata_minus_elemsZmd_arg�partsrw rT �valueZlowercase_headerZis_custom_metar' r' r( rL X s\ �� � � �� z$SetMetaCommand._ParseMetadataHeaders)N) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr r� r� r � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr rO rX rQ rz r r r, rM rL � __classcell__r' r' rY r( r/ � sH ���� � ? 1r/ )N);r� � __future__r r r r r� �apitools.base.pyr �gslib.cloud_apir r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.name_expansionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager Z gslib.utilsr r � gslib.utils.cloud_api_helperr �gslib.utils.metadata_utilr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr r � gslib.utils.translation_helperr r r �PutToQueueWithTimeoutr� r� r� r� rO r) r. r/ r' r' r' r( �<module> sT ��H __pycache__/__init__.cpython-39.pyc 0000644 00000060540 15025166704 0013157 0 ustar 00 a �!c�} � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZ ddl Z ddl Z ddl Z ddl Z ddlZe �d�Zed�d Zed �d Zed �Zd d� Zd dd�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG d d � d e�Z G d d � d e�Z dS )!z< Classes for subcommands of the yum command line interface. � )�print_function)�unicode_literals)� OptionParser)�_N�dnfz+To diagnose the problem, try running: '%s'.z rpm -Va --nofiles --nodigestzDYou probably have corrupted RPMDB, running '%s' might fix the issue.zrpm --rebuilddba You have enabled checking of packages via GPG keys. This is a good thing. However, you do not have any GPG public keys installed. You need to download the keys for packages you wish to install and install them. You can do that by running the command: rpm --import public.gpg.key Alternatively you can specify the url to the key you would like to use for a repository in the 'gpgkey' option in a repository section and {prog} will install it for you. For more information contact your distribution or package provider.c C sj | j js dS | �� sf| j�� D ]F}|js.|jr |js t�dt j t j j d�� t�td�|� t jj�q dS )z�Verify that there are gpg keys for the enabled repositories in the rpm database. :param base: a :class:`dnf.Base` object. :raises: :class:`cli.CliError` Nz %s ��progzProblem repository: %s)�confZgpgcheckZ_gpg_key_check�reposZ iter_enabledZ repo_gpgcheckZgpgkey�loggerZcritical�gpg_msg�formatr �util�MAIN_PROG_UPPERr �cli�CliError)�baser �repo� r �=/usr/lib/python3.9/site-packages/dnf/cli/commands/__init__.py� _checkGPGKey: s r r c C s| | j �� rdS |D ]@}|�d�r2tj�|�r2 dS tjj�|�d }|dv r dS qt d�� d� | j j ��}tj�|��dS )z�Verify that there is at least one enabled repo. :param base: a :class:`dnf.Base` object. :param possible_local_files: the list of strings that could be a local rpms :raises: :class:`cli.CliError`: Nz.rpmr )�http�ftp�file�httpsz*There are no enabled repositories in "{}".z", ")r Z _any_enabled�endswith�os�path�existsr Zpycomp�urlparser r �joinr Zreposdirr r )r Zpossible_local_filesZlfile�scheme�msgr r r �_checkEnabledRepoK s r# c @ s� e Zd ZdZg ZdZdZdd� Zedd� �Z edd � �Z ed d � �Z d d � Z dd� Z dd� Zdd� Zdd� Zdd� Zdd� ZdS )�Commandz%Abstract base class for CLI commands.� Nc C s || _ d S �N�r ��selfr r r r �__init__f s zCommand.__init__c C s | j jS r& )r r �r) r r r r j s z Command.basec C s | j d S )Nr ��aliasesr+ r r r �_basecmdo s zCommand._basecmdc C s | j jjS r& )r r �outputr+ r r r r/ s s zCommand.outputc C s dS )z4Define command specific options and arguments. #:apiNr )r) �parserr r r � set_argparserw s zCommand.set_argparserc C s dS )z*Do any command-specific pre-configuration.Nr r+ r r r � pre_configure{ s zCommand.pre_configurec C s dS )z&Do any command-specific configuration.Nr r+ r r r � configure� s zCommand.configurec C s& t |tjj�rttfS td| ��dS )z.Get suggestions for resolving the given error.zerror not supported yet: %sN)� isinstancer � exceptionsZTransactionCheckError� _RPM_VERIFY�_RPM_REBUILDDB�NotImplementedError)r) �errorr r r �get_error_output� s zCommand.get_error_outputc C s dS )zExecute the command.Nr r+ r r r �run� s z Command.runc C s dS )z$Finalize operation after resolvementNr r+ r r r � run_resolved� s zCommand.run_resolvedc C s dS )z%Finalize operations post-transaction.Nr r+ r r r �run_transaction� s zCommand.run_transaction)�__name__� __module__� __qualname__�__doc__r- �summary�optsr* �propertyr r. r/ r1 r2 r3 r: r; r< r= r r r r r$ _ s$ r$ c @ sR e Zd ZdZdZed�ZdZddddd d d d eh Ze d d� �Z dd� Z dd� Z dS )� InfoCommandzRA class containing methods needed by the cli to execute the info command. )�info�ifz4display details about a package or group of packages�all� available� installed�extras�updates�upgrades� autoremove�recent� obsoletesc C s� |� � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jdd td �| j| jtjtd �d � d S ) N�--all�_packages_action� store_constrH �show all packages (default)��dest�action�const�default�help� --availablerI � show only available packages�rV rW rX rZ � --installedrJ � show only installed packages�--extrasrK �show only extras packages� --updatesrM �show only upgrades packages� --upgrades� --autoremoverN � show only autoremove packages�--recentrO �#show only recently changed packages�packages�*�PACKAGEzPackage name specification��nargs�metavar�choicesrY rW rZ )� add_mutually_exclusive_group� add_argumentr � pkgnarrows�DEFAULT_PKGNARROWr �PkgNarrowCallback)�clsr0 �narrowsr r r r1 � sL ���������zInfoCommand.set_argparserc C s| | j j}d|_| jjr"| jj| j_| jjdkr4d|_| jjrd| jjr\| j �dd| jj � nd| j_| jjdkrxd| j_d S )NTrJ � --obsoletes�--rP rL rM ) r �demands�sack_activationrC rR �packages_action�available_reposrP �_option_conflict�r) ry r r r r3 � s zInfoCommand.configurec C s& | j �| j� | j�d| jj| jj�S )NrF �r � _populate_update_security_filterrC r �output_packagesr{ ri r+ r r r r; � s �zInfoCommand.runN) r> r? r@ rA r- r rB rs rr � classmethodr1 r3 r; r r r r rE � s � rE c @ s$ e Zd ZdZdZed�Zdd� ZdS )� ListCommandzRA class containing methods needed by the cli to execute the list command. )�listZlsz$list a package or groups of packagesc C s& | j �| j� | j�d| jj| jj�S )Nr� r r+ r r r r; � s �zListCommand.runN)r> r? r@ rA r- r rB r; r r r r r� � s r� c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �ProvidesCommandzVA class containing methods needed by the cli to execute the provides command. )�providesZ whatprovidesZprovZwpz*find what package provides the given valuec C s | j ddtd�td�d� d S )N� dependency�+ZPROVIDEz#Provide specification to search for�rm rn rZ �rq r �r0 r r r r1 � s �z ProvidesCommand.set_argparserc C s | j j}d|_d|_d|_d S �NTF)r ry r| Zfresh_metadatarz r~ r r r r3 � s zProvidesCommand.configurec C s t �td�� | j�| jj�S )NzSearching Packages: )r �debugr r r� rC r� r+ r r r r; � s zProvidesCommand.runN� r> r? r@ rA r- r rB � staticmethodr1 r3 r; r r r r r� � s r� c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �CheckUpdateCommandzZA class containing methods needed by the cli to execute the check-update command. )� check-updatez check-upgradez$check for available package upgradesc C s0 | j ddddtd�d� | j ddtd �d � d S ) Nz --changelogs� changelogsF� store_truez show changelogs before update)rV rY rW rZ ri rj rk )rm rn r� r� r r r r1 s �z CheckUpdateCommand.set_argparserc C s6 | j j}d|_d|_d|_| jjr(d|_t| j� d S �NT) r ry rz r| Zplugin_filtering_enabledrC r� r# r r~ r r r r3 s z CheckUpdateCommand.configurec C sR | j j| jdd� | jj| jjd| jjd�}|r:d| j j_| jj j rN| j � � d S )NZgte)Zcmp_typeT)�print_r� �d ) r r� rC r � check_updatesri r� ry �success_exit_statusr Zautocheck_running_kernelZ_check_running_kernel�r) �foundr r r r; s � zCheckUpdateCommand.runNr� r r r r r� � s r� c s e Zd ZdZG dd� de�ZG dd� de�ZG dd� de�ZG dd � d e�ZG d d � d e�Z G d d � d e�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�ZG dd� de�Zeeeee e e e e eeh ZdZed�Z� fdd�Zd d � Zd d � Zd d!� Z� ZS )"�RepoPkgsCommandz2Implementation of the repository-packages command.c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z%RepoPkgsCommand.CheckUpdateSubCommand�'Implementation of the info sub-command.)r� c C s | j j}d|_d|_d S r� )r ry r| rz r~ r r r r3 ( s z/RepoPkgsCommand.CheckUpdateSubCommand.configurec C s* | j j| jj| jdd�}|r&d| jj_dS )�?Execute the command with respect to given arguments *cli_args*.T)r� r� N)r r� rC � pkg_specs�reponamer ry r� r� r r r � run_on_repo- s �z1RepoPkgsCommand.CheckUpdateSubCommand.run_on_repoN�r> r? r@ rA r- r3 r� r r r r �CheckUpdateSubCommand# s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.InfoSubCommandr� )rF c C sh | j j}d|_| jjr"| jj| j_| jjdkr4d|_| jjrd| jjr\| j �dd| jj � nd| j_d S )NTrJ rw rx rP ) r ry rz rC �_pkg_specs_action�pkg_specs_actionr| rP r} r~ r r r r3 9 s z(RepoPkgsCommand.InfoSubCommand.configurec C s. | j �| j� | j�d| jj| jj| j� dS )r� rF N�r r� rC r r� r� r� r� r+ r r r r� F s �z*RepoPkgsCommand.InfoSubCommand.run_on_repoNr� r r r r �InfoSubCommand4 s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z!RepoPkgsCommand.InstallSubCommandz*Implementation of the install sub-command.)�installc C s$ | j j}d|_d|_d|_d|_d S r� �r ry r| rz � resolving� root_userr~ r r r r3 Q s z+RepoPkgsCommand.InstallSubCommand.configurec C s� | j �| j� t| j| j � d}| jjshz| j�d| j� W n$ tj j y` t � t d�� Y q�0 d}nt| jjD ]j}z| j�|| j� W nL tj j y� } z0d�|j| jjj�|��}t � |� W Y d }~qpd }~0 0 d}qp|s�tj �t d���d S )NFrj �No package available.Tz{}: {}z No packages marked for install.)r r� rC r r r� r� r� r r5 � MarkingErrorr rF r r �valuer/ �term�bold�Error)r) �done�pkg_spec�er" r r r r� X s$ z-RepoPkgsCommand.InstallSubCommand.run_on_repoNr� r r r r �InstallSubCommandL s r� c @ s e Zd ZdZdZdd� ZdS )z RepoPkgsCommand.ListSubCommandz'Implementation of the list sub-command.)r� c C s. | j �| j� | j�d| jj| jj| j� dS )r� r� Nr� r+ r r r r� z s �z*RepoPkgsCommand.ListSubCommand.run_on_repoN)r> r? r@ rA r- r� r r r r �ListSubCommandu s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.MoveToSubCommandz*Implementation of the move-to sub-command.)zmove-toc C s$ | j j}d|_d|_d|_d|_d S r� �r ry rz r| r� r� r~ r r r r3 � s z*RepoPkgsCommand.MoveToSubCommand.configurec C s� t | j| j� d}| jjs�z| jjd| jd� W nf tjj yT t � t d�� Y nH tjj yv t � t d�� Y n& tjjy� ds�J d��Y n0 d}n�| jjD ]�}z| jj|| jd� W n� tjj y� t d�}t � ||� Y q� tjj �yd } z^|jD ]J}d }| jj�|�}|�r,t d �| }t d �}t � || jj�|�|� �qW Y d }~q�d }~0 tjj�y� d�s�J d��Y q�0 d}q�|�s�tj�t d ���d S )r� Frj )Z new_reponamezNo package installed.r� �+Only the above marking errors are expected.T�No match for argument: %sr% � (from %s)�%Installed package %s%s not available.N�Nothing to do.)r r r rC r� � reinstallr� r r5 �PackagesNotInstalledErrorr rF r �PackagesNotAvailableErrorr� ri �historyr r/ r� r� r� )r) r� r� r" �err�pkg�xmsg�pkgrepor r r r� � sB 0� z,RepoPkgsCommand.MoveToSubCommand.run_on_repoNr� r r r r �MoveToSubCommand� s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z&RepoPkgsCommand.ReinstallOldSubCommandz0Implementation of the reinstall-old sub-command.)z reinstall-oldc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 � s z0RepoPkgsCommand.ReinstallOldSubCommand.configurec C s� t | j| j� d}| jjs�z| j�d| j| j� W nj tjj yZ t d�}t � |� Y nH tjj y| t � t d�� Y n& tjjy� ds�J d��Y n0 d}n�| jjD ]�}z| j�|| j| j� W n� tjj y� t d�}t � ||� Y q� tjj �yl } z^|jD ]J}d}| jj�|�}|�r4t d �| }t d �}t � || jj�|�|� �q W Y d }~q�d }~0 tjj�y� d�s�J d��Y q�0 d}q�|�s�tj�t d ���d S ) r� Frj �)No package installed from the repository.r� r� Tr� r% r� r� Nr� )r r r rC r� r� r� r r5 r� r r rF r� r� ri r� r r/ r� r� r� )r) r� r" r� r� r� r� r� r r r r� � sH � 0� z2RepoPkgsCommand.ReinstallOldSubCommand.run_on_repoNr� r r r r �ReinstallOldSubCommand� s r� c s4 e Zd ZdZdZ� fdd�Zdd� Zdd� Z� ZS ) z#RepoPkgsCommand.ReinstallSubCommandz,Implementation of the reinstall sub-command.)r� c s, t tj| ��|� t�|�t�|�f| _dS )�Initialize the command.N)�superr� �ReinstallSubCommandr* r� r� �wrapped_commandsr( �� __class__r r r* � s �z,RepoPkgsCommand.ReinstallSubCommand.__init__c C s2 d| j j_| jD ] }| j|_| j|_|�� qd S r� )r ry r| r� rC r� r3 �r) Zcommandr r r r3 � s z-RepoPkgsCommand.ReinstallSubCommand.configurec C sZ t | j| j� | jD ]0}z |�� W n tjjy> Y qY q0 qVqtj�td���dS )r� z!No packages marked for reinstall.N) r r r r� r� r r5 r� r r� r r r r� s z/RepoPkgsCommand.ReinstallSubCommand.run_on_repo) r> r? r@ rA r- r* r3 r� � __classcell__r r r� r r� � s r� c @ s, e Zd ZdZdZdd� Zdd� Zdd� Zd S ) z,RepoPkgsCommand.RemoveOrDistroSyncSubCommandz8Implementation of the remove-or-distro-sync sub-command.)zremove-or-distro-syncc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 s z6RepoPkgsCommand.RemoveOrDistroSyncSubCommand.configurec s� | j jj��� tj�|�}|�| j jj�}| j jj� � �fdd�|� � D �}|s`tj � d|��|� � }| j jj j}|D ]<}|j|j|jd�r�| j jj�|� qx| j jjj||d� qxdS )z;Synchronize a package with another repository or remove it.c s g | ]}� � |��kr|�qS r )r )�.0r� �r� r� r r � <listcomp>" s �zIRepoPkgsCommand.RemoveOrDistroSyncSubCommand._replace.<locals>.<listcomp>zno package matched)�name�arch)� clean_depsN)r r ZsackZ disable_repor �subjectZSubjectZget_best_queryr� rJ r5 r� rI r Z clean_requirements_on_remove�filterr� r� Z_goalZ distupgradeZerase) r) r� r� r� �matchesrJ rI r� �packager r� r �_replace s" �� z5RepoPkgsCommand.RemoveOrDistroSyncSubCommand._replacec C s� t | j| j� d}| jjs\z| �d| j� W n( tjj yT t d�}t � |� Y q�0 d}nP| jjD ]F}z| �|| j� W n* tjj y� t d�}t � ||� Y qd0 d}qd|s�tj� t d���dS )r� Frj r� Tr� r� N)r r r rC r� r� r� r r5 r� r r rF r� �r) r� r" r� r r r r� 0 s$ z8RepoPkgsCommand.RemoveOrDistroSyncSubCommand.run_on_repoN)r> r? r@ rA r- r3 r� r� r r r r � RemoveOrDistroSyncSubCommand s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z+RepoPkgsCommand.RemoveOrReinstallSubCommandz6Implementation of the remove-or-reinstall sub-command.)zremove-or-reinstallc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 R s z5RepoPkgsCommand.RemoveOrReinstallSubCommand.configurec C s t | j| j� d}| jjs�z | jjd| j| jdd� W nH tjj y^ t d�}t � |� Y n& tjj y~ dszJ d��Y n0 d}nz| jjD ]p}z | jj|| j| jdd� W nJ tjj y� t d�}t � ||� Y q� tjj y� ds�J d��Y q�0 d}q�|�stj�t d���d S ) r� Frj T)Z old_reponameZnew_reponame_neqZ remove_nar� z)Only the above marking error is expected.r� r� N)r r r rC r� r� r� r r5 r� r r rF r� r� r� r r r r� Y s8 � � z7RepoPkgsCommand.RemoveOrReinstallSubCommand.run_on_repoNr� r r r r �RemoveOrReinstallSubCommandM s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.RemoveSubCommandz)Implementation of the remove sub-command.)�removec C s* | j j}d|_d|_d|_d|_d|_d S r� )r ry rz Z allow_erasingr| r� r� r~ r r r r3 � s z*RepoPkgsCommand.RemoveSubCommand.configurec C s� d}| j jsPz| j�d| j� W n( tjjyH td�}t � |� Y q�0 d}n^| j jD ]T}z| j�|| j� W n6 tjjy� } zt � t |�� W Y d}~qXd}~0 0 d}qX|s�t � td�� dS )r� Frj r� TNz No packages marked for removal.) rC r� r r� r� r r5 r� r r rF �strZwarning)r) r� r" r� r� r r r r� � s $z,RepoPkgsCommand.RemoveSubCommand.run_on_repoNr� r r r r �RemoveSubCommand~ s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z!RepoPkgsCommand.UpgradeSubCommandz*Implementation of the upgrade sub-command.)�upgradez upgrade-toc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 � s z+RepoPkgsCommand.UpgradeSubCommand.configurec C s� t | j| j� d}| jjs.| j�| j� d}nN| jjD ]D}z| j�|| j� W n& tj j yt t � t d�|� Y q60 d}q6|s�tj �t d���dS )r� FTr� z No packages marked for upgrade.N)r r r rC r� Z upgrade_allr� r� r r5 r� r rF r r� )r) r� r� r r r r� � s z-RepoPkgsCommand.UpgradeSubCommand.run_on_repoNr� r r r r �UpgradeSubCommand� s r� )zrepository-packagesz repo-pkgsz repo-packageszrepository-pkgsz7run commands on top of all packages in given repositoryc s> t t| ��� � � fdd�| jD �}d| _dd� |D �| _dS )r� c 3 s | ]}|� �V qd S r& r �r� �subcmdr' r r � <genexpr>� � z+RepoPkgsCommand.__init__.<locals>.<genexpr>Nc S s i | ]}|j D ] }||�qqS r r, �r� r� �aliasr r r � <dictcomp>� s z,RepoPkgsCommand.__init__.<locals>.<dictcomp>)r� r� r* �SUBCMDSr� �_subcmd_name2obj)r) r Z subcmd_objsr� r' r r* � s �zRepoPkgsCommand.__init__c C s` |� � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jdd tjtd �td �d � d d!� | jD �}d"d!� | jD �}|jd#d d$|d%�|�d&� d}|d dddd'ddh}|jd(d)td*�||tjtd+�d,� d S )-NrQ r� rS rH rT rU r[ rI r\ r] r^ rJ r_ r` rK ra rb rM rc rd re rN rf rg rO rh r� � ZREPOIDz Repository ID)rm rW rn rZ c S s g | ]}|j d �qS )r r, r� r r r r� � r� z1RepoPkgsCommand.set_argparser.<locals>.<listcomp>c S s g | ]}|j D ]}|�qqS r r, r� r r r r� � r� r� Z SUBCOMMANDz, )rm rn ro rZ rP r� rj rk zPackage specificationrl )rp rq r r Z_RepoCallbackEnabler� r rt )r) r0 rv Zsubcommand_choicesZsubcommand_choices_allrs rr r r r r1 � sh �������� � ���z RepoPkgsCommand.set_argparserc C s� z| j | jjd | _W n@ tjjtfyX } z | jj�� tjj�W Y d}~n d}~0 0 | j| j_| jj d | j_ | j� � dS )z8Verify whether the command can run with given arguments.r N) r� rC r� r r r �KeyError� optparser� print_usager� r3 )r) r� r r r r3 s zRepoPkgsCommand.configurec C s | j �� dS )z>Execute the command with respect to given arguments *extcmds*.N)r� r� r+ r r r r; s zRepoPkgsCommand.run)r> r? r@ rA r$ r� r� r� r� r� r� r� r� r� r� r� r� r- r rB r* r1 r3 r; r� r r r� r r� s0 ) 79 >1(#� + r� c @ s0 e Zd ZdZdZed�Zedd� �Zdd� Z dS ) � HelpCommandzRA class containing methods needed by the cli to execute the help command. )rZ z display a helpful usage messagec C s* | j ddtd�td�jtjjd�d� d S )N�cmd�?ZCOMMANDz {prog} command to get help forr r� )rq r r r r r r� r r r r1 s ��zHelpCommand.set_argparserc C sL | j jr| j j| jjvr&| jj�� n"| jj| j j }| jj�|| �� d S r& )rC r� r Z cli_commandsr� � print_helpr� r r r r; $ s �zHelpCommand.runN) r> r? r@ rA r- r rB r� r1 r; r r r r r� s r� )r ) rA Z __future__r r Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.clir Zdnf.exceptionsZ dnf.pycompZdnf.utilZloggingr Z getLoggerr r6 r7 r r r# �objectr$ rE r� r� r� r� r� r r r r �<module> s> ��� 9?$ y __pycache__/stat.cpython-39.pyc 0000644 00000012055 15025166704 0012371 0 ustar 00 a +(Wg� � @ s4 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l m Z dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl m Z ddl!m"Z" dZ#de# d Z$G dd � d e�Z%dS ) zEImplementation of Unix-like stat command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketListingObject)�AccessDeniedException)�EncryptionException)�NotFoundException)�Command)�CommandArgument)� ApiSelector)�CommandException)�InvalidUrlError)�NO_URLS_MATCHED_TARGET)�ContainsWildcard)�StorageUrlFromString)�NO_MAX)�ENCRYPTED_FIELDS)�PrintFullInfoAboutObject)� UNENCRYPTED_FULL_LISTING_FIELDS)�GcloudStorageMapz gsutil stat url... z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> The stat command outputs `object metadata <https://cloud.google.com/storage/docs/metadata>`_ for the specified object URLs. It is similar to running: gsutil ls -L gs://some-bucket/some-object but is more efficient because it avoids listing objects and gets the minimum necessary amount of object metadata. Moreover, because it avoids listing objects (which for some storage providers are eventually consistent) the ``gsutil stat`` command provides a strongly consistent way to check for the existence (and read the metadata) of an object. The ``gsutil stat`` command does, however, make listing object requests if you specify URLs using wildcards. If run with the ``gsutil -q`` option nothing is printed, e.g.: gsutil -q stat gs://some-bucket/some-object This behavior can be useful when writing scripts: even though nothing is printed from the command, it still has an exit status of 0 for an existing object and 1 for a non-existent object. NOTE: Unlike the gsutil ls command, the stat command does not support operations on sub-directories. For example, if you run the command: gsutil -q stat gs://some-bucket/some-subdir/ gsutil looks for information about an object called "some-subdir/" (with a trailing slash) inside the bucket "some-bucket", as opposed to operating on objects nested under gs://some-bucket/some-subdir/. Unless you actually have an object with that name, the operation fails. However, you can use the stat command on objects within subdirectories. For example, this command works as expected: gsutil -q stat gs://some-bucket/some-subdir/file.txt c @ sp e Zd ZdZejdg ededdddej ej gej e � � gd� Z ejdg dd ei d �Zeg d �i d �Zd d� ZdS )� StatCommandz&Implementation of gsutil stat command.�stat� � Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzDisplay object status)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objects�listz --fetch-encrypted-object-hashesz--stat)�gcloud_command�flag_mapc C s� t t }d}| jD �]h}d}t|�}|�� s4td��z�t|�rR| �|�j|d�}nbz"| j j |j |j |j |j|d�}W n0 ty� | j j |j |j |j |jtd�}Y n0 t||d�g}|D ]0}|�� r�|d7 }t�� �tj�r�t|dd� q�W n` t�y& t�� �tj��r"tj�d | d � Y n( t�y: � Y n t�yL Y n0 |st�� �tj��rxtj�t | d � d }q|�r�dS dS ) z%Command entry point for stat command.Fr z,The stat command only works with object URLs)�bucket_listing_fields)� generation�provider�fields)� root_objectr )�incl_aclz+You aren't authorized to read %s - skipping� T) r r �argsr �IsObjectr r �WildcardIterator� IterObjects� gsutil_api�GetObjectMetadata� bucket_name� object_namer3 �schemer r �logging� getLogger� isEnabledFor�INFOr r �sys�stderr�writer r r ) �selfZ stat_fieldsZfound_nonmatching_arg�url_strZ arg_matches�url�blr_iterZ single_obj�blr� rN �"/opt/gsutil/gslib/commands/stat.py� RunCommand� sh �� � �� zStatCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr �gcloud_storage_maprP rN rN rN rO r ] s> ���� � r )&rT � __future__r r r r rB rF �gslib.bucket_listing_refr �gslib.cloud_apir r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r r �gslib.storage_urlr r �gslib.utils.constantsr Zgslib.utils.ls_helperr r r �gslib.utils.shim_utilr rV r\ r rN rN rN rO �<module> s< ��. __pycache__/test.cpython-39.pyc 0000644 00000043351 15025166704 0012400 0 ustar 00 a +(Wg q � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZ ddlZdd lmZ dd lmZ dd lmZ dd lmZ dd l m Z ddl m Z ddl m!Z! ddl m"Z" ddl m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( z ddl)Z)W n e*�yX dZ)Y n0 ej+�rfe,Z-dZ.dZ/dZ0dZ1de1 de. Z2edd�Z3d d � Z4d d � Z5d d!� Z6d"d#� Z7d$d%� Z8d*d&d'�Z9G d(d)� d)e�Z:dS )+�&Implementation of gsutil test command.� )�absolute_import)�print_function)�division)�unicode_literals)� namedtupleN)�range)�ProjectIdException)�Command)�ResetFailureCount)�CommandException)�PopulateProjectId)� GetTestNames)�InvokedFromParFile)�unittest)�NO_MAX)�UTF8)� IS_WINDOWS� �2 �sequential_onlyz3 gsutil test [-l] [-u] [-f] [command command...] z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> The gsutil test command runs the gsutil unit tests and integration tests. The unit tests use an in-memory mock storage service implementation, while the integration tests send requests to the production service using the `preferred API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ set in the boto configuration file. CAUTION: The ``test`` command creates test buckets and objects in your project. Force quitting the ``test`` command can leave behind stale buckets, objects, and HMAC keys in your project. To run both the unit tests and integration tests, run the command with no arguments: gsutil test To run the unit tests only (which run quickly): gsutil test -u Tests run in parallel regardless of whether the top-level -m flag is present. To limit the number of tests run in parallel to 10 at a time: gsutil test -p 10 To force tests to run sequentially: gsutil test -p 1 To have sequentially-run tests stop running immediately when an error occurs: gsutil test -f To run tests for one or more individual commands add those commands as arguments. For example, the following command will run the cp and mv command tests: gsutil test cp mv To list available tests, run the test command with the -l argument: gsutil test -l The tests are defined in the code under the gslib/tests module. Each test file is of the format test_[name].py where [name] is the test name you can pass to this command. For example, running "gsutil test ls" would run the tests in "gslib/tests/test_ls.py". You can also run an individual test class or function name by passing the test module followed by the class name and optionally a test name. For example, to run the an entire test class by name: gsutil test naming.GsutilNamingTests or an individual test function: gsutil test cp.TestCp.test_streaming You can list the available tests under a module or class by passing arguments with the -l option. For example, to list all available test functions in the cp module: gsutil test -l cp To output test coverage: gsutil test -c -p 500 coverage html This will output an HTML report to a directory named 'htmlcov'. Test coverage is compatible with v4.1 of the coverage module (https://pypi.python.org/pypi/coverage). <B>OPTIONS</B> -b Run tests against multi-regional US buckets. By default, tests run against regional buckets in us-central1. -c Output coverage information. -f Exit on first sequential test failure. -l List available tests. -p N Run at most N tests in parallel. The default value is %d. -s Run tests against S3 instead of GS. -u Only run unit tests. �TestProcessDataz name return_code stdout stderrc s G � �fdd�dt j�� � S )z�Creates a closure of CustomTestResult. Args: total_tests: The total number of tests being run. Returns: An instance of CustomTestResult. c s$ e Zd ZdZ�� �fdd�Z� ZS )z3MakeCustomTestResultClass.<locals>.CustomTestResultzDA subclass of unittest.TextTestResult that prints a progress report.c s� t � | ��|� | jr�d�|�� �d�dd � �}d| j�t| j�t| j �t| j �|f }|d d� }|� d�}| j � d| � d S )N�.���z( %d/%d finished - E[%d] F[%d] s[%d] - %s�I z%s - )�super� startTest�dots�join�id�splitZtestsRun�len�errorsZfailures�skipped�ljust�stream�write)�self�testZtest_id�message)�CustomTestResult� __class__� total_tests� �"/opt/gsutil/gslib/commands/test.pyr � s �� z=MakeCustomTestResultClass.<locals>.CustomTestResult.startTest)�__name__� __module__� __qualname__�__doc__r � __classcell__r- �r* r, )r+ r. r* � s r* )r ZTextTestResult)r, r- r4 r. �MakeCustomTestResultClass� s r5 c C sX | g}g }|rT|� � }|D ]6}t|tj�r6|�|� q|�|�� td�d� � qq |S )zGTakes a list of test suites and returns a list of contained test names.�gslib.tests.test_N)�pop� isinstancer � TestSuite�appendr r! )� test_suiteZsuites� test_names�suiter( r- r- r. �GetTestNamesFromSuites� s r> c C s t | j��d�d d | j S )z<Converts a python.unittest to its gsutil test-callable name.�'� r )�strr+ r �_testMethodName)� test_caser- r- r. �TestCaseToName� s �rD c C s ddl m} g }g }g }g }| g}g }|rp|�� }t|tjj�rX|jD ]} |�| � qFq&t|tj �r&|�|� q&|D ]�} t | | j d�} t | dd�r�|�t | �� qtt | dd�s�|�t | �� qtt | dd�s�|�t | �� qtt| |�r�|�t | �� qt|�t | �� qtt |�t |�t |�t |�fS )a� Splits a test suite into groups with different running properties. Args: test_suite: A python unittest test suite. Returns: 4-part tuple of lists of test names: (tests that must be run sequentially, tests that must be isolated in a separate process but can be run either sequentially or in parallel, unit tests that can be run in parallel, integration tests that can run in parallel) r )�GsUtilUnitTestCaseNZrequires_isolationFZis_parallelizableT)Z"gslib.tests.testcase.unit_testcaserE r7 r8 r r= r9 Z_testsr: ZTestCase�getattrrB rD �sorted) r; rE �isolated_tests�sequential_testsZ parallelizable_integration_testsZparallelizable_unit_testsZitems_to_evaluateZcases_to_evaluateZ suite_or_case�itemrC Z test_methodr- r- r. � SplitParallelizableTestSuite� s: �rK c C s d}| D ]}|s|d7 }q|S )z*Counts number of falses in the input list.r r@ r- )Z input_listZ num_falserJ r- r- r. �CountFalseInList s rL c C s� |}t � stjgng }tjjr$dgng }tjjr6dgng } g } zddt� g} W n tyb Y n0 t � � } | } t |�|k �r�|t | �k �r�t j �� } |r�|| d< t� }dd� t|tjg | dg | | d t g | | t d �d � g �D �}t�| �D ] \}}t�|�|t�|�< q�|�tj|tjtj|d �� |d 7 }|�d� t � � | dkrptd|| t |�t | �f � t � � } qp|t | �k�r�tdt |�t | �f � |S )ag Creates test processes to run tests in parallel. Args: parallel_tests: List of all parallel tests. test_index: List index of last created test before this function call. process_list: List of running subprocesses. Created processes are appended to this list. process_done: List of booleans indicating process completion. One 'False' will be added per process created. max_parallel_tests: Maximum number of tests to run in parallel. root_coverage_file: The root .coverage filename if coverage is requested. Returns: Index of last created test. �-s�-bz-oz GSUtil:default_project_id=%sZGSUTIL_COVERAGE_OUTPUT_FILEc S s g | ]}t �|��qS r- )�six� ensure_str)�.0�partr- r- r. � <listcomp>H s z'CreateTestProcesses.<locals>.<listcomp>r( �--r6 N)�stdout�stderr�envr@ Fr z.Created %d new processes (total %d/%d created)z.Test process creation finished (%d/%d created)) r �sys� executable�tests�util� RUN_S3_TESTS�USE_MULTIREGIONAL_BUCKETSr r �timerL r! �os�environ�copy�dict�list�gslib� GSUTIL_PATH�_SEQUENTIAL_ISOLATION_FLAGrO � iteritemsrP r: � subprocess�Popen�PIPE�print)Zparallel_tests� test_index� process_list� process_done�max_parallel_tests�root_coverage_fileZorig_test_indexZexecutable_prefixZ s3_argumentZmultiregional_bucketsZproject_id_argZprocess_create_start_time� last_log_timerW Zenvstr�cmd�k�vr- r- r. �CreateTestProcesses sz � � ��������� �� �� �ru c @ sp e Zd ZdZejdg ededdddegd� Z ej dg dd e i d �Z d d � Z d d� Ze� e� fdd�Zdd� ZdS )� TestCommandr r( r zbuflp:scTF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�supported_private_args� command_helpz2Run gsutil unit/integration tests (for developers))� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s< g }g }g }d}d}d} t |�} t�� } } t|d||||d�} t |�| k �r�tt |��D ]�}|| sZ|| �� du rxqZd||< || �� \}}|| j�� || j�� || j dkr�|d7 }|� t || || j ||d�� qZt |�| k �rt|| ||||d�} t |�| k r@t�� | dk�r�t dt |�| |f � t |�| k�rR|d7 }n t |�} d}|d k�r�g }tt |��D ] }|| �sx|� || � �qxt�� | }t d j tt|��|d �� t�� } t�d� q@t�� }|�r0|D ]J}|jdk�r�|j�d �}t d |j � |D ]}t |�t��� � �q�q�||| fS )aZ Executes the parallel/isolated portion of the test suite. Args: parallel_integration_tests: List of tests to execute. max_parallel_tests: Maximum number of parallel tests to run at once. coverage_filename: If not None, filename for coverage output. Returns: (int number of test failures, float elapsed time) r )rp NTr@ )�name� return_coderU rV r z %d/%d finished - %d failures� zL{sec} seconds elapsed since beginning parallel tests. Still running: {procs})�secZprocs� zResults for failed test %s:)r! r^ ru r �poll� communicaterU �closerV � returncoder: r rk �formatrA �int�sleepr� r r� �decoder �strip)r' �parallel_integration_testsro Zcoverage_filenamerm rn Zprocess_results�num_parallel_failuresZ progress_less_logging_cyclesZcompleted_as_of_last_log�num_parallel_testsZparallel_start_timerq rl Zproc_numrU rV Z still_running�elapsedZprocess_run_finish_time�resultZ new_stderr�liner- r- r. �RunParallelTests� s� ���� �� � �z TestCommand.RunParallelTestsc C s~ t d|| |f � t d|| t|| �|t|�|t|�f � | �|� t � |sb|rbt d� n|rnt d� |szt d� dS )z6Prints test results for parallel and sequential tests.z-Parallel tests complete. Success: %s Fail: %szDRan %d tests in %.3fs (%d sequential in %.3fs, %d parallel in %.3fs)�OKzFAILED (parallel tests)zFAILED (sequential tests)N)rk �float�PrintSkippedTests)r' �num_sequential_tests�sequential_success�sequential_skipped�sequential_time_elapsedr� r� �parallel_time_elapsedr- r- r. �PrintTestResults� s( � ��� z TestCommand.PrintTestResultsc C sh t |�dkst |�dkrdt|�}t|�}|�|�}td� |D ]$\}}td|�� � td| � q>dS )a� Prints all skipped tests, and the reasons they were skipped. Takes the union of sequentual_skipped and parallel_skipped, and pretty-prints the resulting methods and reasons. Note that these two arguments are lists of tuples from TestResult.skipped as described here: https://docs.python.org/2/library/unittest.html#unittest.TestResult.skipped Args: sequentual_skipped: An instance of TestResult.skipped. parallel_skipped: An instance of TestResult.skipped. r zTests skipped:z z Reason: N)r! �set�unionrk r )r' r� Zparallel_skippedZ all_skipped�method�reasonr- r- r. r� � s z TestCommand.PrintSkippedTestsc ) C sn d}d}t }d}d}| jr�| jD ]�\}}|dkr:dtj_q |dkrHd}q |dkrVd}q |dkrdd}q |dt krvd}q |dkr�t|�}q |d kr�tjjs�td ��dtj_ q |d kr dtj_ q |r�t s�td ��tjj r�|t kr�| j �d t � t }tt� �}|�r.| j�s.tdt|� � tdd�t|��� dS | j�r�g } | jD ]>} | |v �sb| �d�d |v �rr| �d| � n | �| � �q@ndd� |D �} t�� t�� } | �rlt�� } | D ]�} z| �| �}| � |� W n� t t f�yf } zzd| t |�f }z t | � W nH t!�yH } z.t"�#� }t$�%dd|�}|d| 7 }W Y d}~n d}~0 0 t|��W Y d}~n d}~0 0 �q�|�r�t&| �}tdt|� � tdd�t|��� dS t'�(� �)� t'j*k�r�d}nd }t'�+t'j,� |�r t j d gd d t-�.� d gd!�}|�/� |�0� d}d}t1| �\}}}}tjj �s&g }t'�2d"|� t'�2d#|� t'�2d$|� t'�2d%|� |�pnt|�dk�on| }d&t3j4d'< |�r�| �5� } t6| �} tj7|| |d(�} | �8| �} | �9� }�nf|dk�r�||7 }g }t:�:� } t|�t|� �rXtd)t|�t|� � || }!| �;td*d� |!D ���} | �5� }"t6|"�} tj7|| |d(�} | �8| �} | �9� }| j<}#nd}"d}t:�:� | }$||7 }t|�}%|%�s�n�g }#t=||%�}&|%dk�r�|dk�r�d+}'|&t k�r�|'d,7 }'td-�t>�?|'|%|&f ��� n td.|% � | �@|||�r�|jAjBnd�\}}(| �C|"||#|$|%||(� |�rJ|�D� |�E� |�F� td/|jAjB � d0t3j4d'< |�rj|�sjtG� dS dS )1z)Command entry point for the test command.FrN Tz-cz-fz-lrT z-prM zbS3 tests require S3 credentials. Please add appropriate credentials to your .boto file and re-run.z-uzrCoverage has been requested but the coverage module was not found. You can install it with "pip install coverage".zCReducing parallel tests to %d due to S3 maximum bucket limitations.zFound %d test names:� z r r �gslib.tests.test_%sc S s g | ] }d | �qS )r� r- )rQ r� r- r- r. rS H � z*TestCommand.RunCommand.<locals>.<listcomp>zLFailed to import test code from file %s. TestLoader provided this error: %sz\nz z Additional traceback: %sNr@ � rd zgslib/third_party/*z gslib/tests/*�*)�sourceZomitzSequential tests to run: %szIsolated tests to run: %sz Parallel unit tests to run: %sz%Parallel integration tests to run: %s�1�GSUTIL_TEST_ANALYTICS)� verbosity� resultclass�failfastz Running %d tests sequentially.c S s g | ]}|�qS r- r- )rQ � test_namer- r- r. rS � r� z1Running %d tests in parallel mode (%d processes).z� Please be patient while your CPU is incinerated. If your machine becomes unresponsive, consider reducing the amount of parallel test processes by running 'gsutil test -p <num_processes>'.� z4Running %d tests sequentially in isolated processes.z%Coverage information was saved to: %s�0)H� _DEFAULT_TEST_PARALLEL_PROCESSES�sub_optsrZ r[ r] rf �longZ HAS_S3_CREDSr r\ ZRUN_INTEGRATION_TESTS�coverage�#_DEFAULT_S3_TEST_PARALLEL_PROCESSES�logger�warnrG r �argsrk r! r r r: r ZinstallHandlerZ TestLoaderr9 ZloadTestsFromNameZaddTests� ImportError�AttributeErrorrA � __import__� Exception� traceback� format_exc�re�subr> �logging� getLogger�getEffectiveLevel�INFO�disable�ERROR�tempfile� gettempdirZerase�startrK �debugr_ r` ZcountTestCasesr5 ZTextTestRunner�runZ wasSuccessfulr^ ZloadTestsFromNamesr# �min�textwrap�wrapr� Z data_files�filenamer� �stop�combine�saver ))r' r� Z list_testsro Zperform_coverager �o�ar< Zcommands_to_testr� �loaderr= � command_nameZsuite_for_current_command�e�msg� stack_trace�errr� Zcoverage_controllerr� r� rI rH Zparallel_unit_testsr� Zrun_tests_sequentiallyr, r� Zrunner�retZsequential_start_timeZsequential_tests_to_runr� r� r� r� � num_processesr) r� r- r- r. � RunCommand sd ��� � "" ��� �� � ��� ���� ��� zTestCommand.RunCommandN)r/ r0 r1 r2 r �CreateCommandSpec� _SYNOPSISr rf � command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� r- r- r- r. rv h s2 � � Zrv )N);r2 � __future__r r r r � collectionsr r� r_ rh r� rX r� r� r^ r� rO � six.movesr rd �gslib.cloud_apir � gslib.commandr r �gslib.exceptionr �gslib.project_idr Z gslib.testsrZ Zgslib.tests.utilr r r �gslib.utils.constantsr r Zgslib.utils.system_utilr r� r� �PY3r� r� r� r� rf r� r� r r5 r> rD rK rL ru rv r- r- r- r. �<module> sr �^��b�5 � N __pycache__/web.cpython-39.pyc 0000644 00000017203 15025166704 0012173 0 ustar 00 a +(Wgx' � @ s( d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z dZ dZ e e � d� Z!dZ"dZ#de" e# Z$ee!e$�Z%ee e#�Z&ee e"�Z'G dd� de �Z(dS )z<Implementation of website configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMapzR gsutil web set [-m <main_page_suffix>] [-e <error_page>] gs://<bucket_name>... z% gsutil web get gs://<bucket_name> � a <B>SET</B> The ``gsutil web set`` command allows you to configure or disable the website configuration on your buckets. The ``set`` sub-command has the following options (omit both options to unset the configuration): <B>SET OPTIONS</B> -m <index.html> Specifies the object name to serve when a bucket listing is requested via a custom domain. -e <404.html> Specifies the error page to serve when a request is made for a non-existent object via a custom domain a4 <B>GET</B> The ``gsutil web get`` command retrieves the web semantics configuration for a bucket and displays a JSON representation of the configuration. In Google Cloud Storage, this would look like the following: { "notFoundPage": "404.html", "mainPageSuffix": "index.html" } ay Cloud Storage allows you to configure a bucket to behave like a static website when the bucket is accessed through a `custom domain <https://cloud.google.com/storage/docs/request-endpoints#custom-domains>`_. For example, if you set a ``main_page_suffix``, a subsequent GET bucket request through a custom domain serves the specified "main" page instead of performing the usual bucket listing. Similarly, if you set an ``error_page``, a subsequent GET object request through a custom domain for a non-existent object serves the specified error page instead of the standard Cloud Storage error. Requests to a bucket through other `endpoints <https://cloud.google.com/storage/docs/request-endpoints>`_ are unaffected by the bucket's website configuration. See `Static website examples and tips <https://cloud.google.com/storage/docs/static-website>`_ for additional examples and information. Notes: 1. Because the main page is only served when a bucket listing request is made via a custom domain endpoint, you can continue to use ``gsutil ls`` as you normally would for bucket listing. 2. The main_page_suffix applies to each subdirectory of the bucket. For example, with the main_page_suffix configured to be index.html, a GET request for ``http://www.example.com`` retrieves ``http://www.example.com/index.html`` but shows ``http://www.example.com`` as the URL to the requester, and a GET request for ``http://www.example.com/photos`` retrieves ``http://www.example.com/photos/index.html`` but shows ``http://www.example.com/photos`` as the URL to the requester. 3. There is just one 404.html page. For example, a GET request for ``http://www.example.com/photos/missing`` retrieves ``http://www.example.com/404.html``, not ``http://www.example.com/photos/404.html``. The web command has two sub-commands: c s� e Zd ZdZejdddgededdddej ej gej e � � ge � d�gd �d � Zejdddgd d eeed �d�Zedeg d�i dd�ii d�Z� fdd�Zdd� Zdd� Zdd� Z� ZS ) � WebCommandz%Implementation of gsutil web command.�web� setwebcfg� getwebcfg� zm:e:F� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz(Set a website configuration for a bucket)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr )�storage�buckets�describezk--format=gsutiljson[key=website_config,empty=' has no website configuration.',empty_prefix_key=storage_url]T)�gcloud_command�flag_map�supports_output_translation�r0 r1 c s| | j d dkrjtdtg d�td�td�d�d�ii d�}d| j v sdd | j v sd|jd jd d g7 _|}ntj}t� �|�S ) Nr r )r- r. �updatez--web-error-pagez--web-main-page-suffix)�-e�-mr3 r5 r6 z--clear-web-error-pagez --clear-web-main-page-suffix)�argsr r r0 r �gcloud_storage_map�super�get_gcloud_storage_args)�selfZset_command_mapr8 �� __class__� �!/opt/gsutil/gslib/commands/web.pyr: � s& ��� � �z"WebCommand.get_gcloud_storage_argsc C s� | j | jd dgd�\}}|jdkr@tj�| jj||jd�� nD|jrt|jj sV|jj rttj�t t � |j��d � ntj�d| � dS )z(Gets website configuration for a bucket.r �website�� bucket_fields�s3)�providerr z!%s has no website configuration. )�GetSingleBucketUrlFromArgr7 �scheme�sys�stdout�write� gsutil_api�XmlPassThroughGetWebsiter@ �mainPageSuffix� notFoundPage�strr � MessageToJson)r; � bucket_url�bucket_metadatar> r> r? �_GetWeb� s" � ����zWebCommand._GetWebc C s� d}d}| j r8| j D ]"\}}|dkr*|}q|dkr|}q| j}tjj||d�}d}|D ]\}| j|dgd�} | D ]B} | j} d}| j�d | � tj|d �} | j j | j | | j dgd � qnqV|s�t tt|� ��d S ) z(Sets website configuration for a bucket.Nr6 r5 )rL rM F�idrA Tz&Setting website configuration on %s...)r@ )rD �fieldsr )�sub_optsr7 �apitools_messages�Bucket� WebsiteValue�GetBucketUrlIterFromArg� storage_url�logger�inforJ � PatchBucket� bucket_namerF r r �list) r; �main_page_suffix� error_page�o�a�url_argsr@ � some_matched�url_str� bucket_iter�blr�urlrQ r> r> r? �_SetWeb� s8 � � zWebCommand._SetWebc C sb | j �d�}| jdd� |dkr(| j}n"|dkr8| j}ntd|| jf ��tj|g| j d� |� S )z(Command entry point for the web command.r T)� check_argsr r zBInvalid subcommand "%s" for the %s command. See "gsutil help web".)� subcommandsrU ) r7 �pop� ParseSubOptsrR rj r � command_namer �LogCommandParamsrU )r; �action_subcommand�funcr> r> r? � RunCommand s ���zWebCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr r8 r: rR rj rs � __classcell__r> r> r< r? r } sR ���� ��� "r ))rw � __future__r r r r rG �apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rV �gslib.utils.constantsr �gslib.utils.shim_utilr r � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripry �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� r r> r> r> r? �<module> s> (�(�* __pycache__/label.cpython-39.pyc 0000644 00000024726 15025166704 0012505 0 ustar 00 a +(Wg\? � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( dZ)dZ*dZ+dZ,d Z-d Z.e)e*�/d � e+�/d � d Z0d e, e- e. Z1ee0e1�Z2ee*e,�Z3ee)e-�Z4ee+e.�Z5G d!d"� d"e�Z6dS )#z<Implementation of label command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�PreconditionException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)� shim_util)�NO_MAX)�UTF8)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�LabelTranslationz< gsutil label set <label-json-file> gs://<bucket_name>... z' gsutil label get gs://<bucket_name> z� gsutil label ch <label_modifier>... gs://<bucket_name>... where each <label_modifier> is one of the following forms: -l <key>:<value> -d <key> z� <B>GET</B> The "label get" command gets the `labels <https://cloud.google.com/storage/docs/tags-and-labels#bucket-labels>`_ applied to a bucket, which you can save and edit for use with the "label set" command. ah <B>SET</B> The "label set" command allows you to set the labels on one or more buckets. You can retrieve a bucket's labels using the "label get" command, save the output to a file, edit the file, and then use the "label set" command to apply those labels to the specified bucket(s). For example: gsutil label get gs://bucket > labels.json Make changes to labels.json, such as adding an additional label, then: gsutil label set labels.json gs://example-bucket Note that you can set these labels on multiple buckets at once: gsutil label set labels.json gs://bucket-foo gs://bucket-bar al <B>CH</B> The "label ch" command updates a bucket's label configuration, applying the label changes specified by the -l and -d flags. You can specify multiple label changes in a single command run; all changes will be made atomically to each bucket. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Add the label "key-foo:value-bar" to the bucket "example-bucket": gsutil label ch -l key-foo:value-bar gs://example-bucket Change the above label to have a new value: gsutil label ch -l key-foo:other-value gs://example-bucket Add a new label and delete the old one from above: gsutil label ch -l new-key:new-value -d key-foo gs://example-bucket <B>CH OPTIONS</B> The "ch" sub-command has the following options -l Add or update a label with the specified key and value. -d Remove the label with the specified key. � z ag Gets, sets, or changes the label configuration (also called the tagging configuration by other storage providers) of one or more buckets. An example label JSON document looks like the following: { "your_label_key": "your_label_value", "your_other_label_key": "your_other_label_value" } The label command has three sub-commands: c @ s� e Zd ZdZejdededdddej ej gej e � d�e � � ge �d�ge � � gd�d� Zejdg d d eeeed �d �Zeeg d �i d�eg d�i d�eg d�edejjd�edejjd�d�d�d �i d�Z dd� Z dd� Z dd� Z dd � Z d d � Z!d S ) � LabelCommandz'Implementation of gsutil label command.�label� zl:d:F� )�set�get�ch) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz8Get, set, or change the label configuration of a bucket.)r r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describezj--format=gsutiljson[key=labels,empty=' has no label configuration.',empty_prefix_key=storage_url,indent=2])�gcloud_command�flag_map)r0 r1 �updatez --labels-file)r0 r1 r5 z--remove-labels)� repeat_typez--update-labels)�-d�-lc C s( | j s| �� | j d �� dkr$dS dS )Nr r r r )�args�$RaiseWrongNumberOfArgumentsException�lower��self� r> �#/opt/gsutil/gslib/commands/label.py�_CalculateUrlsStartArg� s z#LabelCommand._CalculateUrlsStartArgc s� t �j�dk r��� �jd }tj�|�s8td| ��t�|dt ��}|� � � W d � n1 sd0 Y t t ddd�� �fd d ��}d }�jdd� }|D ]*}�j |d gd �}|D ]}d}||� q�q�|s�ttt|� ��dS )z8Parses options and sets labels on the specified buckets.r r z Could not find the file "%s".�rN� r ��tries� timeout_secsc s | j }�j�d| � |jdkr4�jj� ||jd� n�d }d }t�� �}�j�|j�t j krj |j ddgd�\}}|j }i }|jr�t�t�|j��}tdd� t�|�D ��}|�|� t�|�}n t�|�}t|d �} tj|d �}�jj|j|| |jd gd � d S ) N�$Setting label configuration on %s...�s3��provider�labels�metageneration�� bucket_fieldsc s s | ]\}}|d fV qd S �Nr> )�.0�key�_r> r> r? � <genexpr> s zELabelCommand._SetLabel.<locals>._SetLabelForBucket.<locals>.<genexpr>��meta_gen_match�rJ �id�� preconditionsrI �fields)� storage_url�logger�info�scheme� gsutil_api�XmlPassThroughSetTagging�json�loads�GetApiSelectorr �JSON�GetSingleBucketUrlFromArg� url_stringrK rJ r �JsonFromMessage�dict�six� iteritemsr5 � DictToMessager �apitools_messages�Bucket� PatchBucket� bucket_name) �blr�url�labels_messagerK Znew_label_jsonrQ �bucket_metadata� label_jsonZ merged_labelsrX �Z label_textr= r> r? �_SetLabelForBucket� sD � � �� �z2LabelCommand._SetLabel.<locals>._SetLabelForBucketFrV rL T)�lenr9 r: �os�path�isfiler �codecs�openr �readr r �GetBucketUrlIterFromArgr �list) r= Zlabel_filenameZ label_fileru � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr> rt r? � _SetLabel� s$ & * zLabelCommand._SetLabelc s0 i � _ d� _� jr�� jD ]�\}}|dkr^|�d�}t|�dkrJtd| ��|d � j |d < q|dkr�|�d�}t|�dkr�td| ��d � j |< � jd7 _q� �� q� j s�td ��ttd dd �� fd d��}d}� j }|s� � � |D ]&}� � |�} | D ]} d}|| � �q q�|�s,tt t |� ��d S )z;Parses options and changes labels on the specified buckets.r r8 �:r zrFound incorrectly formatted option for "gsutil label ch": "%s". To add a label, please use the form <key>:<value>.r r7 zhFound incorrectly formatted option for "gsutil label ch": "%s". To delete a label, provide only its key.NzAPlease specify at least one label change with the -l or -d flags.rB rC c s2 | j }� j�d| � d }d }� j�|j�tjkr�� j}� j rz� j |j ddgd�\}}|j sz|j }tdd� t�� j�D ��}t�|�}nv� j |j ddgd�\}}|j }i }|j r�t�t�|j ��}t�� j�D ]$\}} | s�||v r�||= q�| ||< q�t�|�}t|d�} tj|d�}� jj|j|| |jd gd � d S ) NrF rJ rK rL c s s | ]\}}|r||fV qd S rN r> )rO �k�vr> r> r? rR Q s zCLabelCommand._ChLabel.<locals>._ChLabelForBucket.<locals>.<genexpr>rS rU rV rW )rZ r[ r\ r^ rb r] r rc � label_changes� num_deletionsrd re rJ rK rg rh ri r rj r` ra rf r rk rl rm rn ) ro rp rq rK Zcorrected_changesrQ rr rs rP �valuerX r< r> r? �_ChLabelForBucket= sL � � � � �z0LabelCommand._ChLabel.<locals>._ChLabelForBucketFT)r� r� �sub_opts�splitrv r � RaiseInvalidArgumentExceptionr r r9 r: r} r r~ ) r= �o�aZ label_splitZ val_splitr� r r� r� r� r� r> r< r? �_ChLabel sR �� �� � 3 zLabelCommand._ChLabelc C sb | j |dgd�\}}|jdkr6t| jj||jd�� n(|jrRttj|jdd�� n td| � dS ) z.Gets and prints the labels for a cloud bucket.rJ rL rG rH T)� pretty_printz %s has no label configuration.N)rd r] �printr^ �XmlPassThroughGetTaggingrJ r rf )r= Z bucket_arg� bucket_urlrr r> r> r? �_GetAndPrintLabel s � � � z LabelCommand._GetAndPrintLabelc C s� | j �d�}| jdd� tj| jd� |dkrNtj|gd� | �| j d � nV|dkrntj|gd� | �� n6|dkr�tj|gd� | �� nt d || j | j f ��dS ) z*Command entry point for the label command.r T)� check_args)r� r )� subcommandsr r zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r9 �pop� ParseSubOptsr �LogCommandParamsr� r� r� r� r � command_name)r= �action_subcommandr> r> r? � RunCommand� s$ ��zLabelCommand.RunCommandN)"�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XMLrc r �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument�MakeNCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specr r r �RepeatFlagType�LIST�DICT�gcloud_storage_mapr@ r� r� r� r� r> r> r> r? r � sv � ����� ������ � C_r )7r� � __future__r r r r rz r` rw rh �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rk Z gslib.utilsr �gslib.utils.constantsr r �gslib.utils.retry_utilr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�_CH_DESCRIPTION�lstripr� � _DESCRIPTIONr� r� r� r� r r> r> r> r? �<module> s` �� � � � __pycache__/help.cpython-39.pyc 0000644 00000014645 15025166704 0012355 0 ustar 00 a +(Wg�$ � @ s$ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl m Z ddlZdd lmZ dd lmZ ddlZdd lmZ dd lmZ dd lmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z dZ de d Z!dZ"G dd� de�Z#dS )�&Implementation of gsutil help command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�PIPE)�Popen)�Command)� OLD_ALIAS_MAP)�CommandException)� HelpProvider)�MAX_HELP_NAME_LEN)� constants)� IS_WINDOWS)�IsRunningInteractively)� GetTermLines)� text_utilz" gsutil help [command or topic] z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The following command provides a summary of gsutil commands and additional topics on which help is available: gsutil help The following command provides help about the specified command or topic: gsutil help [command or topic] The following command provides help about the specified sub-command: gsutil help [command] [sub-command] For example, running the following provides help about the "set" sub-command of the "acl" command: gsutil help acl set If you set the PAGER environment variable to the path to a pager program (such as /bin/less on Linux), long help sections are piped through the specified pager. z�Usage: gsutil [-D] [-DD] [-h header]... [-i service_account] [-m] [-o section:flag=value]... [-q] [-u user_project] [command [opts...] args...]c @ s` e Zd ZdZejdddgeddddd dd � Zejddgd d e i d �Z dd� Z dd� Z dd� Z dS )� HelpCommandr �help�?Zmanr � � TF)�command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg� command_helpz"Get help about commands and topics)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s< | � � \}}g }| js�|�dt � dtt� d }t|d dd� d�D ] }|�||jj|jj f � qH|�d� t|d d d� d�D ] }|�||jj|jj f � q�|�d � �ntd }| jd }||vr�|�d| � �nL|| }d}t | j�dk�r�|jj } | �r8| jd | v �r8|d | jd }| | jd } nbd}| �sV|�d||f � nDg } | D ]} | �d|| f � �q^|�d| jd ||d� | �f � |�s(|�s�|jj}|jj } |�d� |�d||jj f � |�d� |�| �d�� t�|dg�d } | �r(d|| f }|�d� |�|� | �d � |�� d S ) z)Command entry point for the help command.z%s Available commands: z %-zs%s r c S s | j jS �N�� help_specr! ��hp� r, �"/opt/gsutil/gslib/commands/help.py�<lambda>r � z(HelpCommand.RunCommand.<locals>.<lambda>)�keyz Additional help topics: �additional_helpc S s | j jS r' r( r* r, r, r- r. w r/ z6 Use gsutil help <command or topic> for detailed help.Fr zNo help available for "%s"N� � Tz`The "%s" command has no subcommands. You can ask for the full help by running: gsutil help %s z gsutil help %s %sz�Subcommand "%s" does not exist for command "%s". You can either ask for the full help about the command by running: gsutil help %s Or you can ask for help about one of the subcommands: %s� z <B>NAME</B> z %s - %s z zr The "%s" alias is deprecated, and will eventually be removed completely. Please use the "%s" command instead.z <B>DEPRECATION WARNING</B> r )� _LoadHelpMaps�args�append�top_level_usage_string�strr �sortedr) r! r$ �lenr& �joinr% �stripr �get� _OutputHelp)�self� help_type_map� help_name_map�output� format_str� help_provZinvalid_subcommand�argr! Zsubcommand_mapr% Zsubcommand_examples� subcommandZ new_aliasZdeprecation_warningr, r, r- � RunCommandj s� � � � � ��� �� � � zHelpCommand.RunCommandc C s t s t� s4t�dd|�}t�dd|�}t�|� dS t�dd|�}t�dd|�}t|�d��}dtj v �r|t � k�rtj d �d �}|d � d �r�|� d � z2t jr�|�tj�}n|}t|td d�j|d� W n< t�y } z"tdd �|�|f ��W Y d}~n d}~0 0 n t�|� dS )z�Outputs simply formatted string. This function paginates if the string is too long, PAGER is defined, and the output is a tty. Args: help_str: String to format. z<B>r z</B>Nz[1mz[0;0mr4 ZPAGERr3 r Zlessz-rT)�stdin�universal_newlines)�inputz Unable to open pager (%s): %s)r r �re�subr � print_to_fdr; �split�os�environr �endswithr7 �six�PY2�encoder �UTF8r r � communicate�OSErrorr r< )r@ Zhelp_str� num_linesZpagerZinput_for_pager�er, r, r- r? � s4 �� � zHelpCommand._OutputHelpc C s� t �tjj�D ]\}}}td| � qt �tjj�D ]\}}}td| � q4i }i }tjjD ] }g ||< q\t � t � � t � � �D ]P}|t u r�q~tj�||� |||jj< |jjD ] }|||< q�||jj �|� q~||fS )a. Returns tuple of help type and help name. help type is a dict with key: help type value: list of HelpProviders help name is a dict with key: help command name or alias value: HelpProvider Returns: (help type, help name) zgslib.commands.%szgslib.addlhelp.%s)�pkgutil� iter_modules�gslib�commands�__path__� __import__Zaddlhelp� help_provider�ALL_HELP_TYPES� itertools�chainr �__subclasses__r � SanityCheckr) r! r" r# r7 )r@ �_� module_namerA rB �srE r" r, r, r- r5 � s& � zHelpCommand._LoadHelpMapsN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXTr) rH r? r5 r, r, r, r- r Q s. � � D%r )$rm � __future__r r r r rc rP r[ rL rS � subprocessr r Zgslib.addlhelpr] � gslib.commandr r �gslib.commands�gslib.exceptionr �gslib.help_providerr r Z gslib.utilsr Zgslib.utils.system_utilr r r r ro rr r8 r r, r, r, r- �<module> s@ �� � __pycache__/versioning.cpython-39.pyc 0000644 00000013013 15025166704 0013574 0 ustar 00 a +(Wg]! � @ s, d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de e! Z"ee e"�Z#de�$� d d d Z%ee e!�Z&ee e �Z'G dd � d e �Z(d S ) z?Implementation of versioning configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz8 gsutil versioning set (on|off) gs://<bucket_name>... z/ gsutil versioning get gs://<bucket_name>... � z� <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable versioning for the specified bucket(s). z� <B>GET</B> The "get" sub-command gets the versioning configuration for a bucket and displays whether or not it is enabled. z� The `Versioning Configuration <https://cloud.google.com/storage/docs/object-versioning>`_ feature enables you to configure a Google Cloud Storage bucket to keep old versions of objects. The gsutil versioning command has two sub-commands: z'--format=value[separator=""](name.sub("z", "gs://").sub("$", ": "),zversioning.enabledz .yesno("Enabled", "Suspended"))c @ s� e Zd ZdZejdddgededdddej ej gej e dd d gd �e � � ge � � gd �d � Z ejdddgddeeed�d�Zeeddddegi dd�eeg d�i d�eg d�i d�d�i d�d�i d�Zd d � Zd d � Zd d!� Zd"d#� Zd$S )%�VersioningCommandz,Implementation of gsutil versioning command.� versioning� setversioning� getversioning� � F�mode�on�off)�choices)�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz4Enable or suspend versioning for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r0 r1 �updatez --versioning)r3 r4 )r0 r1 r6 z--no-versioning�r r c C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r= �(/opt/gsutil/gslib/commands/versioning.py�_CalculateUrlsStartArg� s z(VersioningCommand._CalculateUrlsStartArgc C s� | j d �� }|dvr$td| j ��| j dd� }|s>| �� d}|D ]�}| j|dgd�}|D ]p}|j}d }tjtj� � d �}|d kr�| j � d |� d |j _ n| j � d |� d|j _ | jj|j||jdgd� q^qF|s�ttt|� ��dS )z+Gets versioning configuration for a bucket.r r7 z,Argument to "%s set" must be either <on|off>r8 NF�id�� bucket_fieldsT)r r z Enabling versioning for %s...z Suspending versioning for %s...)�provider�fields)r9 r; r � command_namer: �GetBucketUrlIterFromArg� storage_url�apitools_messages�Bucket�VersioningValue�logger�infor �enabled� gsutil_api� PatchBucket� bucket_name�schemer r2 ) r< �versioning_arg�url_args� some_matched�url_str� bucket_iter�blr�url�bucket_metadatar= r= r> �_SetVersioning� s: �� � z VersioningCommand._SetVersioningc C s� | j }d}|D ]^}| j|dgd�}|D ]D}d}|jjrV|jjjrVtd|j�d� � q&td|j�d� � q&q|s�tt t |� ��dS ) z6Gets versioning configuration for one or more buckets.Fr rA Tz %s: Enabled�/z %s: SuspendedN) r9 rF � root_objectr rM �print� url_string�rstripr r r2 )r< rS rT rU rV rW r= r= r> �_GetVersioning� s �z VersioningCommand._GetVersioningc C s� | j �d�}|dkr*| j}tj|gd� nL|dkr`| j}| j d �� }|dv rvtj||gd� ntd|| j| jf ��|� dS )z/Command entry point for the versioning command.r r )� subcommandsr r7 zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r9 �popr` r �LogCommandParamsrZ r; r rE )r< �action_subcommand�funcrR r= r= r> � RunCommand� s$ � ��z VersioningCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr? rZ r` rf r= r= r= r> r N st �������� � ����%�)!r N))rj � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rH �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprl �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrr �get_format_flag_caretrv rs rt r r= r= r= r> �<module> sL �� ���� __pycache__/autoclass.cpython-39.pyc 0000644 00000014015 15025166704 0013412 0 ustar 00 a +(Wg�# � @ s4 d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de! e Z"ee e"�Z#ee e �Z$ee e!�Z%de�&� d d d d d d Z'G d d � d e�Z(dS ) z5This module provides the autoclass command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)� text_util)�NO_MAX)�GcloudStorageMap)� shim_utilz7 gsutil autoclass set (on|off) gs://<bucket_name>... z. gsutil autoclass get gs://<bucket_name>... � z� <B>SET</B> The ``set`` sub-command requires an additional sub-command, either ``on`` or ``off``, which enables or disables Autoclass for the specified bucket(s). a+ <B>GET</B> The ``get`` sub-command gets the current Autoclass configuration for a bucket. The returned configuration has the following fields: ``enabled``: a boolean field indicating whether the feature is on or off. ``toggleTime``: a timestamp indicating when the enabled field was set. z� The `Autoclass <https://cloud.google.com/storage/docs/autoclass>`_ feature automatically selects the best storage class for objects based on access patterns. This command has two sub-commands, ``get`` and ``set``. z--format=value[separator="z"](zformat("gs://{}:", name),z format(" Enabled: {}",z&autoclass.enabled.yesno(True, False)),z format(" Toggle Time: {}",zautoclass.toggleTime))c @ s� e Zd ZdZejdg ededdddej gej e � d�ge ddd gd �e � � gd �d � Z ejdg d deeed �d�Zeedddedgi dd�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd� Zd d � Zd d � Zd d!� Zd"S )#�AutoclassCommandz(Implements the gsutil autoclass command.� autoclass� � F� �mode�on�off)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzConfigure Autoclass feature)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r/ r0 �updatez--enable-autoclass)r2 r3 )r/ r0 r5 z--no-enable-autoclass)r r c C sn |j }| jj|jdg|jd�}t|��d�}|jrPt|jdd�}t|jdd�}nd}d}t d� |||�� dS ) z(Gets the autoclass setting for a bucket.r ��fields�provider�/�enabledF� toggleTimeNz#{}: Enabled: {} Toggle Time: {}) � storage_url� gsutil_api� GetBucket� bucket_name�scheme�str�rstripr �getattr�print�format)�self�blr� bucket_url�bucket_metadata�bucketr: Z toggle_time� rK �'/opt/gsutil/gslib/commands/autoclass.py�_get_autoclass� s ��z AutoclassCommand._get_autoclassc C s^ |j }tj�� }|dk|_tj|d�}td|t|��d�f � | jj |j |dg|j d� dS )z'Turns autoclass on or off for a bucket.r )r zSetting Autoclass %s for %sr9 r r6 r ) r< �apitools_messages�Bucket�AutoclassValuer: rD rA rB r= � PatchBucketr? r@ )rF rG � setting_argrH Zautoclass_configrI rK rK rL �_set_autoclass� s � �z AutoclassCommand._set_autoclassc C s& | j �d�}|dvr td��d}g }d}|dkr8| j}n0|dkrh| j}| j �d�}t�|d� |�|� | j� d�t j kr�td � t �d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j� | jj�t j kr�td � t �d | j ����d }|| g|�R � q�q�|�s"ttt|� ��dS )z3Handles autoclass command on Cloud Storage buckets.r r z autoclass only supports get|setNr r z-Only on and off values allowed for set option�gsr z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS Buckets.T)�args�popr rM rS r � InsistOnOrOff�appendr= �GetApiSelectorr �JSON�join�textwrap�wrap� command_name�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr< r@ r r1 ) rF Z subcommandZsubcommand_funcZsubcommand_argsrR Z some_matched�url_args�url_strZ bucket_iter�bucket_listing_refrK rK rL � _autoclass� sV � �� ����zAutoclassCommand._autoclassc C sZ | j d }| jdd� |dks&|dkrJtj| jd� tj|gd� | �� S td| ��d S ) z.Command entry point for the autoclass command.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rU � ParseSubOptsr �LogCommandParamsrf rd r )rF �action_subcommandrK rK rL � RunCommand� s �zAutoclassCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rZ r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_maprM rS rd rk rK rK rK rL r R sp ����� �� � ����%�)/r ))ro � __future__r r r\ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rN Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.shim_utilr r � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprq �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrv rx rw �get_format_flag_newlinerz r rK rK rK rL �<module> sX �� ������� __pycache__/ls.cpython-39.pyc 0000644 00000045471 15025166704 0012044 0 ustar 00 a +(Wg�j � @ s~ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' ddl(m)Z) e�*d �Z+d Z,d e, d Z-G d d!� d!e.�Z/G d"d#� d#e �Z0dS )$zCImplementation of Unix-like ls command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�NotFoundException)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)�NO_MAX)�S3_DELETE_MARKER_GUID)�UTF8)�ENCRYPTED_FIELDS)�LsHelper)�PrintFullInfoAboutObject)� UNENCRYPTED_FULL_LISTING_FIELDS)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� text_util)�AclTranslation)�LabelTranslation)�MakeHumanReadablez([^\s]*)\s([^\.\+]*).*z? gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Retrieves a list of providers, buckets, or objects matching the criteria, ordered in the list lexicographically by name. <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> If you run ``gsutil ls`` without URLs, it lists all of the Google Cloud Storage buckets under your default project ID (or all of the Cloud Storage buckets under the project you specify with the ``-p`` flag): gsutil ls If you specify one or more provider URLs, ``gsutil ls`` lists buckets at each listed provider: gsutil ls gs:// gsutil currently supports ``gs://`` and ``s3://`` as valid providers If you specify bucket URLs, or use `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_ to capture a set of buckets, ``gsutil ls`` lists objects at the top level of each bucket, along with the names of each subdirectory. For example: gsutil ls gs://bucket might produce output like: gs://bucket/obj1.htm gs://bucket/obj2.htm gs://bucket/images1/ gs://bucket/images2/ The "/" at the end of the last 2 URLs tells you they are subdirectories, which you can list using: gsutil ls gs://bucket/images* If you specify object URLs, ``gsutil ls`` lists the specified objects. For example: gsutil ls gs://bucket/*.txt lists all files whose name matches the above wildcard at the top level of the bucket. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_. <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> Listing a bucket or subdirectory (as illustrated near the end of the previous section) only shows the objects and names of subdirectories it contains. You can list all objects in a bucket by using the -r option. For example: gsutil ls -r gs://bucket lists the top-level objects and buckets, then the objects and buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. If you want to see all objects in the bucket in one "flat" listing use the recursive ("**") wildcard, like: gsutil ls -r gs://bucket/** or, for a flat listing of a subdirectory: gsutil ls -r gs://bucket/dir/** If you want to see only the subdirectory itself, use the -d option: gsutil ls -d gs://bucket/dir <B>LISTING OBJECT DETAILS</B> If you specify the -l option, gsutil outputs additional information about each matching provider, bucket, subdirectory, or object. For example: gsutil ls -l gs://bucket/*.html gs://bucket/*.txt prints the object size, creation time stamp, and name of each matching object, along with the total count and sum of sizes of all matching objects: 2276224 2020-03-02T19:25:17Z gs://bucket/obj1.html 3914624 2020-03-02T19:30:27Z gs://bucket/obj2.html 131 2020-03-02T19:37:45Z gs://bucket/obj3.txt TOTAL: 3 objects, 6190979 bytes (5.9 MiB) Note that the total listed in parentheses above is in mebibytes (or gibibytes, tebibytes, etc.), which corresponds to the unit of billing measurement for Google Cloud Storage. You can get a listing of all the objects in the top-level bucket directory (along with the total count and sum of sizes) using a command like: gsutil ls -l gs://bucket To print additional detail about objects and buckets use the gsutil ls -L option. For example: gsutil ls -L gs://bucket/obj1 prints something like: gs://bucket/obj1: Creation time: Fri, 26 May 2017 22:55:44 GMT Update time: Tue, 18 Jul 2017 12:31:18 GMT Storage class: STANDARD Content-Length: 60183 Content-Type: image/jpeg Hash (crc32c): zlUhtg== Hash (md5): Bv86IAzFzrD1Z2io/c7yqA== ETag: 5ca67960a586723b7344afffc81 Generation: 1378862725952000 Metageneration: 1 ACL: [ { "entity": "project-owners-867484910061", "projectTeam": { "projectNumber": "867484910061", "team": "owners" }, "role": "OWNER" }, { "email": "jane@gmail.com", "entity": "user-jane@gmail.com", "role": "OWNER" } ] TOTAL: 1 objects, 60183 bytes (58.77 KiB) Note that results may contain additional fields, such as custom metadata or a storage class update time, if they are applicable to the object. Also note that some fields, such as update time, are not available with the (non-default) XML API. See also "gsutil help acl" for getting a more readable version of the ACL. <B>LISTING BUCKET DETAILS</B> If you want to see information about the bucket itself, use the -b option. For example: gsutil ls -L -b gs://bucket prints something like: gs://bucket/ : Storage class: STANDARD Location constraint: US Versioning enabled: False Logging configuration: None Website configuration: None CORS configuration: None Lifecycle configuration: None Requester Pays enabled: True Labels: None Default KMS key: None Time created: Thu, 14 Jan 2016 19:25:17 GMT Time updated: Thu, 08 Jun 2017 21:17:59 GMT Metageneration: 1 Bucket Policy Only enabled: False ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Default ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Note that some fields above (time created, time updated, metageneration) are not available with the (non-default) XML API. <B>OPTIONS</B> -l Prints long listing (owner, length). -L Prints even more detail than -l. Note: If you use this option with the (non-default) XML API it generates an additional request per object being listed, which makes the -L option run much more slowly and cost more than the default JSON API. -d List matching subdirectory names instead of contents, and do not recurse into matching subdirectories even if the -R option is specified. -b Prints info about the bucket when used with a bucket URL. -h When used with -l, prints object sizes in human readable format (e.g., 1 KiB, 234 MiB, 2 GiB, etc.) -p proj_id Specifies the project ID or project number to use for listing buckets. -R, -r Requests a recursive listing, performing at least one listing operation per subdirectory. If you have a large number of subdirectories and do not require recursive-style output ordering, you may be able to instead use wildcards to perform a flat listing, e.g. ``gsutil ls gs://mybucket/**``, which generally performs fewer listing operations. -a Includes non-current object versions / generations in the listing (only useful with a versioning-enabled bucket). If combined with -l option also prints metageneration for each listed object. -e Include ETag in long listing (-l) output. c @ s e Zd ZdZdZdZdZdS )� ListingStylez(Enum class for specifying listing style.�SHORT�LONG� LONG_LONGN)�__name__� __module__� __qualname__�__doc__r r r � r# r# � /opt/gsutil/gslib/commands/ls.pyr s r c @ s� e Zd ZdZejdddgededdddej ej gej e � � gd � Z ejdddgd d ei d �Zeg d �ed�ed�ed�ed�ed�ed�ed�ed�ed�d� d�Zdd� Zdd� Zd d � Zd S ) � LsCommandz$Implementation of gsutil ls command.�ls�dir�listr z aebdlLhp:rRFT) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz#List providers, buckets, or objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storager& z --fetch-encrypted-object-hashes�-r�-l�-L�-b�-e�-az--readable-sizesz --project) r< �-Rr= r> r? r@ rA �-h�-p)�gcloud_command�flag_mapc C s� |t jks|t jkr"t�|� dS |j}|j}|j}|j||t � |j �t � |j �|j o\|j j|jrfdnd|jrrdnd|jr~dnd|jr�dnd|jo�|jjd� }|jr�d|d< |jr�tj |jdd�|d< nd|d< |jr�|jjr�|jj|d < nd|d < |jr�dnd|d < |j�r*|jj�r*|jj� d �|d < |j �r<|j |d < |j �rP|j j |d< |j �rb|j |d< |j!�rz|j!� d�|d< |j"�r�|j"� d�|d< |j#�r�|j#|d< |j$�r�|j$j%�r�|j$j%j}||d< |j$j&�r�|j$j&|d< |j'�r�|j'|d< |j(�r|j(|d< |D ]R}|| } t)| t*j+��rd| v�r.�q| �,dd�} | �-d��sNd| } | ||< �qd} d} d} d}d}d}d}d}d}d}d}d}d |v �r�d} d |v �r�d } d|v �r�d } d|v �r�d }d|v �r�d }d|v �r�d }d|v �r�d!}d|v �r�d"}d|v �rd#}d|v �rd$}d|v �r$d%}d|v �r2d&}t�d'| d( | d) | | d* d+ | | | | | | | | d, j.f i |��� |j/j0d-k�r�t�d.� dS )/a Print listing info for given bucket. Args: bucket_blr: BucketListingReference for the bucket being listed listing_style: ListingStyle enum describing type of output desired. Returns: Tuple (total objects, total bytes) in the bucket. NZPresent�None) �bucket� storage_class�location_constraint�aclZ default_acl� versioning�website_config�logging_config� cors_config�lifecycle_config�requester_paysZretention_policyT)� pretty_print�labelsZdefault_kms_key�encryption_configz %a, %d %b %YZautoclass_enabled_dateZ location_typeZcustom_placement_locations�metagenerationz%a, %d %b %Y %H:%M:%S GMT� time_created�updatedZdefault_eventbased_holdZbucket_policy_only_enabledZpublic_access_prevention�rpoZ satisfies_pzs� z � z2 Autoclass: Enabled on {autoclass_enabled_date} z" Location type: {location_type} z4 Placement locations: {custom_placement_locations} z$ Metageneration: {metageneration} z Time created: {time_created} z Time updated: {updated} z5 Default Event-Based Hold: {default_eventbased_hold} z' Retention Policy: {retention_policy} z: Bucket Policy Only enabled: {bucket_policy_only_enabled} z6 Public access prevention: {public_access_prevention} z RPO: {rpo} z" Satisfies PZS: {satisfies_pzs} z-{bucket} : Storage class: {storage_class} z- Location constraint: {location_constraint} z� Versioning enabled: {versioning} Logging configuration: {logging_config} Website configuration: {website_config} CORS configuration: {cors_config} Lifecycle configuration: {lifecycle_config} Requester Pays enabled: {requester_pays} z Labels: {labels} z% Default KMS key: {default_kms_key} z, ACL: {acl} Default ACL: {default_acl}�s3z�Note: this is an S3 bucket so configuration values may be blank. To retrieve bucket configuration values, use individual configuration commands such as gsutil acl get <bucket>.)1r r r r � print_to_fd� root_object�location� storageClass� url_stringr �JsonFromMessagerK �defaultObjectAclrL �enabled�website�logging�cors� lifecycle�billing� requesterPays�retentionPolicyrS r � encryption�defaultKmsKeyName� autoclass� toggleTime�strftime� locationType�customPlacementConfig� dataLocationsrU � timeCreatedrW �defaultEventBasedHold�iamConfiguration�bucketPolicyOnly�publicAccessPreventionrX � satisfiesPZS� isinstance�six� string_types�replace� startswith�format� storage_url�scheme)�selfZ bucket_blr� listing_stylerH rJ rI �fieldsrc �keyZprevious_value� new_valueZautoclass_lineZlocation_type_lineZ custom_placement_locations_lineZmetageneration_lineZtime_created_lineZtime_updated_lineZ default_eventbased_hold_lineZretention_policy_lineZ bucket_policy_only_enabled_lineZ public_access_prevention_lineZrpo_lineZsatisifies_pzs_liner# r# r$ �_PrintBucketInfo^ s2 � � � � � � �� � � � ���� � � � � � � ����������zLsCommand._PrintBucketInfoc C s� |j }|j}|jr4t|jjv r4d}d}d}|d7 }n$| jrDt|j�nt|j�}|j}d}t � dt|j ��}d}d} d} | j r�|d7 }t|j �} | jr�|d 7 }|j} |||| | d �} t�|| � ||fS ) z(Prints an object with ListingStyle.LONG.�0r z<DeleteMarker>� z\1T\2Zz"%(size)10s %(timestamp)s %(url)sNz# metageneration=%(metageneration)sz etag=%(etag)s)�size� timestamp�urlrU �etag)r] r` �metadatar �additionalProperties�human_readabler r� �str�JSON_TIMESTAMP_RE�subrs � all_versionsrU � include_etagr� r r\ ) r� �bucket_listing_ref�obj�url_str� size_string� num_bytes�num_objsr� ZprintstrZ encoded_etagZencoded_metagenZ format_argsr# r# r$ �_PrintLongListing� sB � � � �zLsCommand._PrintLongListingc s� d}d}t j}d}d� _d� _d� _d� _d� _� jr� jD ]�\}}|dkrTd� _q<|dkrdd� _q<|dkrrd}q<|dkr�d� _q<|dkr�t j}q<|dkr�t j }q<|d kr�t |d � |� _ q<|d ks�|d kr�d� _q<|d kr<d� _q<� j s�dg� _ d}d}� fdd�} | } � j D �]>} t | �} | �� �r6td� j ��d} |t jk�sR|t jk�rZdg} n|t j k�rng d�} | �� �r�� �d| j �j| d�D ]}� �||� �q��q| �� �r.|�r.d}� �| �j| d�D ]F}t| ��s�|j�s�� jj|jj dg| jd� � �||� |d7 }�q�t| ��sN|�sNd}�qdd� }|t jk�rbt � j� j � j| � j� jd �}n�|t jk�r�g d �}� j�r�|� d d g� � j�r�|� d � t � j� j � j!|| � j� j|� jd!� }nF|t j k�r t"t# }t � j� j t$|| � j� j|� jd!� }n td"| ��|�%| �\}}}| �&� �r>|dk�r>|dk�r>d}||7 }||7 }�q|�r�|t jk�r�t'�(d#||t)t*|��f � |�r�td$��|�r�t+d%��dS )&z'Command entry point for the ls command.FTrA r@ r? rC r= r> rD z/Invalid non-ASCII character found in project IDr< rB z-dzgs://r c s( t � j�dkr$t�dt�| j� � d S )Nr� z%s:)�len�argsr r\ rz � ensure_textr` ��blr�r� r# r$ �MaybePrintBucketHeaderG s z4LsCommand.RunCommand.<locals>.MaybePrintBucketHeaderz$Only cloud URLs are supported for %sN�id)rK rm rh rf rq rb rk ru rS r^ rp re rg rU rj rt rX rx r_ rs rW rL rd z%s://*)� bucket_fields)r� �providerr� c S s t �ddt�| j�f � d S )Nz%-33s%srZ )r r\ rz r� r` r� r# r# r$ �_PrintPrefixLong� s �z.LsCommand.RunCommand.<locals>._PrintPrefixLong)r� �print_bucket_header_func�should_recurse�list_subdir_contents)�namer� rs rW � generationrU r� )�print_object_func�print_dir_funcr� r� r� r� r� zUnknown listing style: %sz TOTAL: %d objects, %d bytes (%s)z$One or more URLs matched no objects.z+One or more bucket URLs matched no buckets.),r r �recursion_requestedr� r� r� r� �sub_optsr r r � project_idr� r � IsFileUrlr � command_name� IsProvider�WildcardIteratorr� � IterBucketsr� �IsBucketr r] � gsutil_api� GetBucketr � bucket_namer �logger�extend�appendr� r r r �ExpandUrlAndPrint�IsObjectr r\ r �floatr )r� �got_nomatch_errorsZgot_bucket_nomatch_errorsr� Zget_bucket_info�o�aZ total_objs� total_bytesr� Zprint_bucket_headerr� r r� r� Z total_bucketsr� �listing_helper�bucket_listing_fieldsZexp_dirs�exp_objs� exp_bytesr# r� r$ � RunCommand s � � ��� � � � � � �� � ��zLsCommand.RunCommandN)r r r! r" r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr� r� r� r# r# r# r$ r% ' s^ ������ �� $r% )1r" � __future__r r r r �rerz �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r �gslib.utils.constantsr r r Zgslib.utils.ls_helperr r r r �gslib.utils.shim_utilr r Zgslib.utils.text_utilr Z gslib.utilsr � gslib.utils.translation_helperr r �gslib.utils.unit_utilr �compiler� r� r� �objectr r% r# r# r# r$ �<module> sJ �� i __pycache__/rewrite.cpython-39.pyc 0000644 00000040174 15025166704 0013102 0 ustar 00 a +(Wg�f � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' ddl(m)Z) ddl(m*Z* ddl(m+Z+ ddl(m,Z, ddl-m.Z. dd l-m/Z/ dd l0m1Z1 dd l2m3Z3 dd l2m4Z4 dd l5m6Z6 dd!l7m8Z8 d"Z9d#Z:d$e: d% Z;d&d'� Z<d2d(d)�Z=d*d+� Z>G d,d-� d-e?�Z@G d.d/� d/e�ZAd0d1� ZBdS )3zIImplementation of rewrite command (in-place cloud object transformation).� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�config)�EncryptionException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�FileProgressCallbackHandler)�StorageUrlFromString)�storage_v1_messages)� FileMessage)�GetCloudApiInstance)�NO_MAX)�UTF8)� CryptoKeyType)�CryptoKeyWrapperFromKey)�GetEncryptionKeyWrapper)�MAX_DECRYPTION_KEYS)�GcloudStorageFlag)�GcloudStorageMap)� StdinIterator)� ConvertRecursiveToFlatWildcard)�NormalizeStorageClass)� text_util)�PreconditionsFromHeaders�A z� gsutil rewrite -k [-O] [-f] [-r] [-s] url... gsutil rewrite -k [-O] [-f] [-r] [-s] -I gsutil rewrite -s [-k] [-O] [-f] [-r] url... gsutil rewrite -s [-k] [-O] [-f] [-r] -I z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The gsutil rewrite command rewrites cloud objects, applying the specified transformations to them. The transformation(s) are atomic for each affected object and applied based on the input transformation flags. Object metadata values are preserved unless altered by a transformation. At least one transformation flag, -k or -s, must be included in the command. The -k flag is supported to add, rotate, or remove encryption keys on objects. For example, the command: gsutil rewrite -k -r gs://bucket updates all objects in gs://bucket with the current encryption key from your boto config file, which may either be a base64-encoded CSEK or the fully-qualified name of a Cloud KMS key. The rewrite command acts only on live object versions, so specifying a URL with a generation number fails. If you want to rewrite a noncurrent version, first copy it to the live version, then rewrite it, for example: gsutil cp gs://bucket/object#123 gs://bucket/object gsutil rewrite -k gs://bucket/object You can use the -s option to specify a new storage class for objects. For example, the command: gsutil rewrite -s nearline gs://bucket/foo rewrites the object, changing its storage class to nearline. If you specify the -k option and you have an encryption key set in your boto configuration file, the rewrite command skips objects that are already encrypted with the specified key. For example, if you run: gsutil rewrite -k -r gs://bucket and gs://bucket contains objects encrypted with the key specified in your boto configuration file, gsutil skips rewriting those objects and only rewrites objects that are not encrypted with the specified key. This avoids the cost of performing redundant rewrite operations. If you specify the -k option and you do not have an encryption key set in your boto configuration file, gsutil always rewrites each object, without explicitly specifying an encryption key. This results in rewritten objects being encrypted with either the bucket's default KMS key (if one is set) or Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to determine whether the operation is redundant (and thus skippable) because gsutil cannot be sure how the object is encrypted after the rewrite. Note that if your goal is to encrypt objects with a bucket's default KMS key, you can avoid redundant rewrite costs by specifying the bucket's default KMS key in your boto configuration file; this allows gsutil to perform an accurate comparison of the objects' current and desired encryption configurations and skip rewrites for objects already encrypted with that key. If have an encryption key set in your boto configuration file and specify multiple transformations, gsutil only skips those that would not change the object's state. For example, if you run: gsutil rewrite -s nearline -k -r gs://bucket and gs://bucket contains objects that already match the encryption configuration but have a storage class of standard, the only transformation applied to those objects would be the change in storage class. You can pass a list of URLs (one per line) to rewrite on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to rewrite objects identified by a program, such as: some_program | gsutil -m rewrite -k -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. The rewrite command requires OWNER permissions on each object to preserve object ACLs. You can bypass this by using the -O flag, which causes gsutil not to read the object's ACL and instead apply the default object ACL to the rewritten object: gsutil rewrite -k -O -r gs://bucket <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when rewriting multiple objects. If some of the objects could not be rewritten, gsutil's exit status is non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rewrite ...". -I Causes gsutil to read the list of objects to rewrite from stdin. This allows you to run a program that generates the list of objects to rewrite. -k Rewrite objects with the current encryption key specified in your boto configuration file. The value for encryption_key may be either a base64-encoded CSEK or a fully-qualified KMS key name. If no value is specified for encryption_key, gsutil ignores this flag. Instead, rewritten objects are encrypted with the bucket's default KMS key, if one is set, or Google-managed encryption, if no default KMS key is set. -O When a bucket has uniform bucket-level access (UBLA) enabled, the -O flag is required and skips all ACL checks. When a bucket has UBLA disabled, the -O flag rewrites objects with the bucket's default object ACL instead of the existing object ACL. This is needed if you do not have OWNER permission on the object. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents to be rewritten recursively. -s <class> Rewrite objects using the specified storage class. c C s( | j s| j�t|�� | jd7 _dS )z9Simple exception handler to allow post-completion status.� N)�continue_on_error�logger�error�str�op_failure_count)�cls�e� r* �%/opt/gsutil/gslib/commands/rewrite.py�_RewriteExceptionHandler� s r, c C s | j ||d� d S )N�� thread_state)� RewriteFunc)r( �name_expansion_resultr. r* r* r+ �_RewriteFuncWrapper� s r1 c c s. | D ]$}t |�jdur"td| ��|V qdS )zAGenerator function that ensures generation-less (live) arguments.Nz-"rewrite" called on URL with generation (%s).)r � generationr )�url_strs�url_strr* r* r+ �GenerationCheckGenerator� s �r5 c @ s e Zd ZdZdZdZdS )�_TransformTypesz Enum class for valid transforms.� crypto_key� storage_classN)�__name__� __module__� __qualname__�__doc__� CRYPTO_KEY� STORAGE_CLASSr* r* r* r+ r6 � s r6 c @ s� e Zd ZdZejdg ededdddej gej e � � gd� Z ej dddgd d ei d �Zeg d �ed �ed�e�ddd�rzdned�ed�ed�ed�ed�d�d�Zdd� Zdd� Zd d d �ZdS ) �RewriteCommandz)Implementation of gsutil rewrite command.�rewriter zfkIrROs:F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZrekey�rotate� command_helpzRewrite objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objects�update�-Iz--continue-on-error�GSUtil�encryption_keyNz--clear-encryption-keyz--no-preserve-acl�-r�-s)rW �-f�-k�-OrZ �-Rr[ )�gcloud_command�flag_mapc C s |j dkrtdt|� ��d S )N�gsz5"rewrite" called on URL with unsupported provider: %s)�schemer r& )�self�urlr* r* r+ � CheckProvider s �z RewriteCommand.CheckProviderc C s� | j | _i | _d| _d| _d| _ddg| _t� | _d| _ t t �| _ | j rP| j j nd| _| jr�| jD ]�\}}|dkrzd| _qb|dkr�| j�tj� qb|dkr�d| _qb|d kr�d| _qb|d ks�|d kr�d| _d| _qb|dkrb| j�tj� t|�| _qb| j�r| j�r td ��t� }n| j�s"td ��| j}| j�s>td| j ��t| j�pJi �| _ t |�}| j�rht |�}t | j | j!| j"| j#|| j| j$| j�p�| j ddgd� }d}| j�s�t |�}t%| j | j!| �&� || j| j| j$d�}t'dt(�D ]R}|d } t)t �*ddt+| � d��} | du �r �q2| j,t-j.k�r�| | j| j < �q�| jdu�rL| j | j| j< | j du �rzd�/t0�1d��} t2d| t3j4d� | j5t6|t7| j dg|d� | j �r�| j �r�d nd } td | j | | f ��dS ) z,Command entry point for the rewrite command.NFr] r[ r r\ TrW r^ rZ r_ z&No arguments allowed with the -I flag.z:The rewrite command (without -I) expects at least one URL.zgrewrite command requires at least one transformation flag. Currently supported transformation flags: %s�name�size)� project_idr# �bucket_listing_fields)� all_versionsri r" rX zdecryption_key%s� a> NOTE: No encryption_key was specified in the boto configuration file, so gsutil will not provide an encryption key in its rewrite API requests. This will decrypt the objects unless they are in buckets with a default KMS key set, in which case the service will automatically encrypt the rewritten objects with that key.z%s )�filer' )� fail_on_error� shared_attrs�seek_ahead_iterator�s� z*%d file%s/object%s could not be rewritten.)8�parallel_operationsr# �csek_hash_to_keywrapper�dest_storage_class�no_preserve_acl�read_args_from_stdinZ supported_transformation_flags�set�transform_typesr' r r � boto_file_encryption_keywrapper�crypto_key_sha256Zboto_file_encryption_sha256�sub_opts�addr6 r= �recursion_requestedrk r> r �argsr r r �headers� preconditionsr5 r r � command_name�debugr$ � gsutil_apiri r �GetSeekAheadGsutilApi�ranger r �getr& � crypto_typer �CSEK�join�textwrap�wrap�print�sys�stderr�Applyr1 r, ) rd �o�ar3 Zurl_strs_generator�name_expansion_iteratorrp Zseek_ahead_url_strs�i� key_number� keywrapper�msg� plural_strr* r* r+ � RunCommand s� � � �� � � � � ��� �zRewriteCommand.RunCommandc C s t | |d�}|j}| �|� |j|j|j|j|jd�}| jrDg |_ n|j sVt d| ��|j rb|j nd }d }|j r�|j j r�|j j }|�d�}|d up�|d u}d } | jd ur�| jjtjkr�| jj} d } | jd ur�| jjtjkr�| jj} | jd u} || ko�|| k} tj| jv�r | �s td||f ��g } tj| jv �rN| jt |j �k�rN| � d� tj| jv �rr| �rr| �rr| � d� t | �t | j�k�r�| j �!d|| f � d S t"�#t$j%t"�&|��}d |_d |_'d |_ d |_ tj| jv �r�| j|_ | d u�r�| |_ d }|d u�r&|| j(v �r| j(| }ntd ||f ��d }tj| jv �rr|�rP| �rP| �srd }n"|�rb| �sbd }n|�sr| �rrd }t)j*�+t,||j-�� t)j*�.� |j/�0t1|d t2�2� d|j3t1j4d�� t5|j/||d�j6}|j7|||j| j8||| j|jg d� |j/�0t1|d t2�2� d|j3t1j4d�� d S )Nr- )r2 �providera No OWNER permission found for object %s. If your bucket has uniform bucket-level access (UBLA) enabled, include the -O option in your command to avoid this error. If your bucket does not use UBLA, you can use the -O option to apply the bucket's default object ACL when rewriting.�asciiz�The "-k" flag was not passed to the rewrite command, but the encryption_key value in your boto config file did not match the key used to encrypt the object "%s" (hash: %s). To encrypt the object using a different key, you must specify the "-k" flag.z storage classzencryption keyz3Skipping %s, all transformations were redundant: %szOMissing decryption key with SHA256 hash %s. No decryption key matches object %sZ RewritingZRotatingZ DecryptingZ EncryptingF)�finishedrh � message_type)�src_url�operation_name)�src_generationr� �progress_callback�decryption_tuple�encryption_tupler� �fieldsT)9r �expanded_storage_urlrf �GetObjectMetadata� bucket_name� object_namer2 rc rv �aclr � kmsKeyName�customerEncryption� keySha256�encoderz r� r �CMEKr7 r� r{ r6 r= ry r r> ru r � storageClass�append�lenr$ �infor �PyValueToMessage�apitools_messages�Object�MessageToPyValue�idrt r� r� �write�_ConstructAnnounceText� url_string�flush� status_queue�putr �timerh � FILE_REWRITEr �call� CopyObjectr� )rd r0 r. r� Z transform_urlZ src_metadataZsrc_encryption_kms_keyZsrc_encryption_sha256Zsrc_was_encryptedZdest_encryption_kms_keyZdest_encryption_sha256Zshould_encrypt_destZencryption_unchangedZredundant_transformsZ dest_metadata�decryption_keywrapperr� r� r* r* r+ r/ � s ������ � � �� ���� �� �� � � ��� � ���� ��zRewriteCommand.RewriteFunc)N)r9 r: r; r< r �CreateCommandSpec� _SYNOPSISr r �JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r r r� �gcloud_storage_maprf r� r/ r* r* r* r+ r? � sN �� �� r? c C sz | dd� � d�}t|�}td�}|t|� | tkr\td�}d|t| | | d� }d||f }dt }|�|�S ) a� Constructs announce text for ongoing operations on url_string. This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and informs the rewrite-related operation ('Encrypting', 'Rotating', or 'Decrypting'). Args: operation_name: String describing the operation, i.e. 'Rotating' or 'Encrypting'. url_string: String describing the file/object being processed. Returns: Formatted announce text for outputting operation progress. N� � z: z...z...%sz%s%s:z{0:%ds})�ljustr� � MAX_PROGRESS_INDICATOR_COLUMNS�format)r� r� Zjustified_op_stringZ start_lenZend_lenZ ellipsis_lenZbase_announce_text� format_strr* r* r+ r� E s ��� r� )N)Cr< � __future__r r r r r� r� r� �apitools.base.pyr �botor �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.name_expansionr r �gslib.progress_callbackr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r � gslib.utils.encryption_helperr r r r �gslib.utils.shim_utilr r Zgslib.utils.system_utilr Zgslib.utils.text_utilr r Z gslib.utilsr � gslib.utils.translation_helperr r� r� r� r, r1 r5 �objectr6 r? r� r* r* r* r+ �<module> sb ��v w __pycache__/hmac.cpython-39.pyc 0000644 00000030621 15025166704 0012325 0 ustar 00 a +(Wg�B � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z dZ dZ dZ dZ dZ!dZ"dZ#dZ$dZ%dZ&e e �'d � e �'d � e �'d � e!�'d � d Z(d d �)e"e#e$e%e&g� Z*ee(e*�Z+d d gZ,d!Z-ee e"�Z.ee e#�Z/ee e$�Z0ee e%�Z1ee!e&�Z2d"d#� Z3d$d%� Z4d&e �5� d' d( d) Z6d&e �5� d* d+ d, d- d. d/ d0 d1 d2 Z7d3Z8ed4�Z9ed5d6d7e6gd8e9id9�Z:eg d:�d8e9id9�Z;ed5d6d;e7gd8e9id9�Z<ed5d6d<e8ged=�ed>�e9d?�d9�Z=ed5d6d<e7ged=�ed@�ed>�e9dA�d9�Z>ed5d6dBe7gedCdDdE��edF�e9dG�d9�Z?G dHdI� dIe�Z@dJS )Kz� Implementation of HMAC key management command for GCS. NOTE: Any modification to this file or corresponding HMAC logic should be submitted in its own PR and release to avoid concurrency issues in testing. � )�absolute_import)�division)�print_function)�unicode_literals)�Command)�CommandArgument)� ApiSelector��CommandException)�CreateHelpText)�LogCommandParams)�PopulateProjectId)�GetCloudApiInstance)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� shim_utilz= gsutil hmac create [-p <project>] <service_account_email> z1 gsutil hmac delete [-p <project>] <access_id> z. gsutil hmac get [-p <project>] <access_id> zJ gsutil hmac list [-a] [-l] [-p <project>] [-u <service_account_email>] zR gsutil hmac update -s (ACTIVE|INACTIVE) [-e <etag>] [-p <project>] <access_id> a <B>CREATE</B> The ``hmac create`` command creates an HMAC key for the specified service account: gsutil hmac create test.service.account@test_project.iam.gserviceaccount.com The secret key material is only available upon creation, so be sure to store the returned secret along with the access_id. <B>CREATE OPTIONS</B> The ``create`` sub-command has the following option -p <project> Specify the ID or number of the project in which to create a key. a� <B>DELETE</B> The ``hmac delete`` command permanently deletes the specified HMAC key: gsutil hmac delete GOOG56JBMFZX6PMPTQ62VD2 Note that keys must be updated to be in the ``INACTIVE`` state before they can be deleted. <B>DELETE OPTIONS</B> The ``delete`` sub-command has the following option -p <project> Specify the ID or number of the project from which to delete a key. a� <B>GET</B> The ``hmac get`` command retrieves the specified HMAC key's metadata: gsutil hmac get GOOG56JBMFZX6PMPTQ62VD2 Note that there is no option to retrieve a key's secret material after it has been created. <B>GET OPTIONS</B> The ``get`` sub-command has the following option -p <project> Specify the ID or number of the project from which to get a key. a� <B>LIST</B> The ``hmac list`` command lists the HMAC key metadata for keys in the specified project. If no project is specified in the command, the default project is used. <B>LIST OPTIONS</B> The ``list`` sub-command has the following options -a Show all keys, including recently deleted keys. -l Use long listing format. Shows each key's full metadata excluding the secret. -p <project> Specify the ID or number of the project from which to list keys. -u <service_account_email> Filter keys for a single service account. a� <B>UPDATE</B> The ``hmac update`` command sets the state of the specified key: gsutil hmac update -s INACTIVE -e M42da= GOOG56JBMFZX6PMPTQ62VD2 Valid state arguments are ``ACTIVE`` and ``INACTIVE``. To set a key to state ``DELETED``, use the ``hmac delete`` command on an ``INACTIVE`` key. If an etag is set in the command, it will only succeed if the provided etag matches the etag of the stored key. <B>UPDATE OPTIONS</B> The ``update`` sub-command has the following options -s <ACTIVE|INACTIVE> Sets the state of the specified key to either ``ACTIVE`` or ``INACTIVE``. -e <etag> If provided, the update will only be performed if the specified etag matches the etag of the stored key. -p <project> Specify the ID or number of the project in which to update a key. � z z� You can use the ``hmac`` command to interact with service account `HMAC keys <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. The ``hmac`` command has five sub-commands: �INACTIVE�ACTIVEz%a, %d %b %Y %H:%M:%S GMTc C s t d| ||f �S )NzD%s %s requires an Access ID to be specified as the last argument. %sr )� command_name� subcommand�synopsis� r �"/opt/gsutil/gslib/commands/hmac.py�_AccessIdException� s ��r c C s� d dd�}d| j }||d| j�7 }||d| j�7 }||d| j�7 }||d| j�t��7 }||d | j�t��7 }||d | jd d �7 }|S )z4Format the key metadata for printing to the console.Tc S s& d}d|| d |f }|r"|d7 }|S )z=Format the metadata name-value pair into two aligned columns.� z %-*s %s�:r r )�name�value�new_line�widthZinfo_strr r r � FormatInfo� s z&_KeyMetadataOutput.<locals>.FormatInfozAccess ID %s: �StatezService Account�Projectz Time CreatedzTime Last UpdatedZEtagF)r )T) �accessId�state�serviceAccountEmail� projectId� timeCreated�strftime� _TIME_FORMAT�updated�etag)�metadatar" �messager r r �_KeyMetadataOutput� s � �r0 z--format=value[separator="z"](z-format("Access ID: {}", metadata.accessId),z"format("Secret: {}", secret))z%"](format("Access ID {}:", accessId),z,format(" State: {}", state),z:format(" Service Account: {}", serviceAccountEmail),z0format(" Project: {}", projectId),z$format(" Time Created: {}",z6 timeCreated.date(format="%a %d %b %Y %H:%M:%S GMT")),z$format(" Time Last Updated: {}",z2 updated.date(format="%a %d %b %Y %H:%M:%S GMT")),z+format(" Etag: {}", etag))zW--format=table[no-heading](format("{} ", accessId),state:width=11, serviceAccountEmail)z --project�storage�hmac�create�-p��gcloud_command�flag_map)r1 r2 �delete�describe�listz--allz--service-account)�-a�-ur4 z--long)r; �-lr< r4 �updatez --activatez --deactivate)r r z--etag)�-s�-er4 c s� e Zd ZdZejddddddejgejee � � ge � � ge � � ge � � ge � � gd�d� Z ej dg d d e eeeeed�d �Z� fd d �Zddd�Zd dd�Zd dd�Zd dd�Zd dd�Zdd� Z� ZS ) � HmacCommandz&Implementation of gsutil hmac command.r2 � � z ae:lp:s:u:T�r3 r8 �getr: r> ) �min_args�max_args�supported_sub_args� file_url_ok�urls_start_arg�gs_api_support�gs_default_api�usage_synopsis�argparse_arguments� command_helpz-CRUD operations on service account HMAC keys.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc sN | j d dkr*d| j v r*tdtii d�}nttttttd�i d�}t� � |�S )Nr r: r= r5 )r3 r8 r> rE r: ) �argsr �LIST_COMMAND_LONG_FORMAT�CREATE_COMMAND�DELETE_COMMAND�UPDATE_COMMAND� GET_COMMAND� LIST_COMMAND�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__r r r^ < s ��� z#HmacCommand.get_gcloud_storage_argsNc C sx | j r| j d | _nd}t|| j| jtf ��t| |d�}|j| j| jdd�}t dd|j j f � t dd|j f � d S ) z'Creates HMAC key for a service account.r zI%s %s requires a service account to be specified as the last argument. %s�� thread_state�gs��providerz%-12s %sz Access ID:zSecret:N) rV �service_account_emailr r �action_subcommand�_CREATE_SYNOPSISr � CreateHmacKey� project_id�printr. r% �secret)r_ rd �err_msg� gsutil_api�responser r r �_CreateHmacKeyP s �� �zHmacCommand._CreateHmacKeyc C sD | j r| j d }nt| j| jt��t| |d�}|j| j|dd� dS )zDeletes an HMAC key.r rc re rf N)rV r r ri �_DELETE_SYNOPSISr � DeleteHmacKeyrl )r_ rd � access_idrp r r r �_DeleteHmacKeyd s � zHmacCommand._DeleteHmacKeyc C sP | j r| j d }nt| j| jt��t| |d�}|j| j|dd�}tt |�� dS )z!Gets HMAC key from its Access Id.r rc re rf N) rV r r ri � _GET_SYNOPSISr � GetHmacKeyrl rm r0 �r_ rd ru rp rq r r r � _GetHmacKeyp s � zHmacCommand._GetHmacKeyc C s� | j r td| j| jtf ��t| |d�}|j| j| j| j dd�}d}| j rh|D ]}t t |�� t � qNn"|D ] }t ||j |j|jf � qldS )z1Lists HMAC keys for a project or service account.z'%s %s received unexpected arguments. %src re rf z %s %-12s %sN)rV r r ri �_LIST_SYNOPSISr � ListHmacKeysrl rh �show_all� long_listrm r0 r% r& r' )r_ rd rp rq Zshort_list_format�itemr r r � _ListHmacKeys~ s* �� � �zHmacCommand._ListHmacKeysc C s� | j s td| j| jtf ��n | j tvr:tdd�t� ��| jrL| jd }nt| j| jt��t | |d�}|j | j || j | j dd�}t t|�� dS ) zUpdate an HMAC key's state.z*A state flag must be supplied for %s %s %sz&The state flag value must be one of %sz, r rc re rf N)r& r r ri �_UPDATE_SYNOPSIS�_VALID_UPDATE_STATES�joinrV r r � UpdateHmacKeyrl r- rm r0 ry r r r �_UpdateHmacKey� s. �� � � �zHmacCommand._UpdateHmacKeyc C sF | j jdd�tjkr td��| j�d�| _| jdd� t | j d� d| _ d| _ d | _ d | _d| _| j r�| j D ]p\}}|d kr�|| _ ql|d kr�t|d � || _ql|d kr�|| _ ql|dkr�d| _ ql|dkr�d| _ql|dkrl|| _ql| js�td�| _| j| j| j| j| jd�}| j|v�r(td| j| jf ��t | jgd� || j � dS )z)Command entry point for the hmac command.re rf z9The "hmac" command can only be used with the GCS JSON APIr T)� check_args)�sub_optsNFr< r4 z/Invalid non-ASCII character found in project IDr? r; r= r@ rD zCInvalid subcommand "%s" for the %s command. See "gsutil help hmac".)� subcommands)rp �GetApiSelectorr �JSONr rV �popri � ParseSubOptsr r� rh r& r} r~ r- r rl r rr rv rz r� r� r )r_ �o�aZmethod_for_argr r r � RunCommand� sT � � � zHmacCommand.RunCommand)N)N)N)N)N) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpecr r� � _SYNOPSISr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_create_help_text�_delete_help_text�_get_help_text�_list_help_text�_update_help_text� help_specr^ rr rv rz r� r� r� � __classcell__r r ra r rA sL ���� rA N)Ar� � __future__r r r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr Z gslib.metricsr �gslib.project_idr � gslib.utils.cloud_api_helperr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr Z gslib.utilsr rj rs rw r{ r� Z_CREATE_DESCRIPTIONZ_DELETE_DESCRIPTION�_GET_DESCRIPTIONZ_LIST_DESCRIPTIONZ_UPDATE_DESCRIPTION�lstripr� r� � _DESCRIPTIONr� r� r+ r� r� r� r� r� r r0 �get_format_flag_newlineZ_CREATE_COMMAND_FORMATZ_DESCRIBE_COMMAND_FORMATZ_LIST_COMMAND_SHORT_FORMATZ _PROJECT_FLAGrX rY r[ r\ rW rZ rA r r r r �<module> s� ������ ���� �������� �� � ���� � �� �� ��� __pycache__/config.cpython-39.pyc 0000644 00000121737 15025166704 0012673 0 ustar 00 a +(Wg�� � @ s@ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZ ddlmZ ddlZdd lmZ ddlZdd lmZ dd lmZ dd l m Z dd l m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl*m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 dZ5d e5 d Z6d Z7d Z8e0� \Z9Z:e9�r�d Z;d!Z<ne=e�>� d"�Z;d#Z<d$Z?d%Z@d&ZAd'ZBd(ZCd)ZDd*ZEd+ZFd,ZGd-e,e-e+e.e(jHe;e<e?e@e?e@eCe eeDeEd.� ZId/ZJG d0d1� d1e�ZKd2d3� ZLdS )4zJImplementation of config command for creating a gsutil configuration file.� )�absolute_import)�print_function)�division)�unicode_literalsN)�input)�ResponseNotReady)�Provider)�Command)�!DEFAULT_TASK_ESTIMATION_THRESHOLD)�MAX_COMPOSE_ARITY)� CredTypes��AbortException)�CommandException)�'CheckAndMaybePromptForAnalyticsEnabling)�RegisterSignalHandler)� constants)� system_util)�CHECK_HASH_ALWAYS)� CHECK_HASH_IF_FAST_ELSE_FAIL)� CHECK_HASH_IF_FAST_ELSE_SKIP)�CHECK_HASH_NEVER)� ShouldProhibitMultiprocessing)�ServerNotFoundError)� HAS_CRYPTOz/ gsutil [-D] config (-a|-e) [-n] [-o <file>] z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The ``gsutil config`` command generally applies to users who have legacy standalone installations of gsutil. If you installed gsutil via the Cloud SDK, ``gsutil config`` fails unless you are specifically using the ``-a`` flag or have configured gcloud to not pass its managed credentials to gsutil (via the command ``gcloud config set pass_credentials_to_gsutil false``). For all other use cases, Cloud SDK users should use the ``gcloud auth`` group of commands instead, which configures OAuth2 credentials that gcloud implicitly passes to gsutil at runtime. To check if you are using gsutil from the Cloud SDK or as a legacy standalone, use ``gsutil version -l`` and in the output look for "using cloud sdk". Important: The default behavior for the ``gsutil config`` command is to obtain user account credentials for authentication. However, user account credentials are no longer supported for standalone gsutil. For this reason, running the default ``gsutil config`` command fails, and using any of the following flags causes the command to fail: ``-b``, ``-f``, ``-r``, ``--reauth``, ``-s``, ``-w``. When using standalone gsutil, it's recommended that you use service account credentials via the ``-e`` flag. The ``gsutil config`` command obtains access credentials for Cloud Storage and writes a `boto/gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ containing the obtained credentials along with a number of other configuration- controllable values. Unless specified otherwise (see OPTIONS), the configuration file is written to ~/.boto (i.e., the file .boto under the user's home directory). If the default file already exists, an attempt is made to rename the existing file to ~/.boto.bak; if that attempt fails the command exits. A different destination file can be specified with the ``-o`` option (see OPTIONS). Because the boto configuration file contains your credentials you should keep its file permissions set so no one but you has read access. (The file is created read-only when you run ``gsutil config``.) <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> Service accounts are useful for authenticating on behalf of a service or application (as opposed to a user). If you use gsutil as a legacy stand-alone tool, you configure credentials for service accounts using the ``-e`` option: gsutil config -e Note that if you use gsutil through the Cloud SDK, you instead activate your service account via the `gcloud auth activate-service-account <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ command. When you run ``gsutil config -e``, you are prompted for the path to your private key file and, if not using a JSON key file, your service account email address and key file password. To get this data, follow the instructions on `Service Accounts <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_. Using this information, gsutil populates the "gs_service_key_file" attribute in the boto configuration file. If not using a JSON key file, gsutil also populates the "gs_service_client_id" and "gs_service_key_file_password" attributes. Note that your service account is NOT considered an Owner for the purposes of API access (see "gsutil help creds" for more information about this). See https://developers.google.com/identity/protocols/OAuth2ServiceAccount for further information on service account authentication. If you want to use credentials based on access key and secret (the older authentication method before OAuth2 was supported), see the ``-a`` option in the OPTIONS section. If you wish to use gsutil with other providers (or to copy data back and forth between multiple providers) you can edit their credentials into the [Credentials] section after creating the initial boto configuration file. <B>OPTIONS</B> -a Prompt for Google Cloud Storage access key and secret (the older authentication method before OAuth2 was supported) instead of obtaining an OAuth2 token. -e Prompt for service account credentials. This option requires that ``-a`` is not set. -n Write the configuration file without authentication configured. This flag is mutually exclusive with all flags other than ``-o``. -o <file> Write the configuration to <file> instead of ~/.boto. Use ``-`` for stdout. z)https://cloud.google.com/console#/projectaY # This file contains credentials and other configuration information needed # by the boto library, used by gsutil. You can edit this file (e.g., to add # credentials) but be careful not to mis-edit any of the variable names (like # "gs_access_key_id") or remove important markers (like the "[Credentials]" and # "[Boto]" section delimiters). # � � � � �0Z50MZ150MZ200M� Z2G� aK [Boto] # http_socket_timeout specifies the timeout (in seconds) used to tell httplib # how long to wait for socket timeouts. The default is 70 seconds. Note that # this timeout only applies to httplib, not to httplib2 (which is used for # OAuth2 refresh/access token exchanges). #http_socket_timeout = 70 # The following two options control the use of a secure transport for requests # to S3 and Google Cloud Storage. It is highly recommended to set both options # to True in production environments, especially when using OAuth2 bearer token # authentication with Google Cloud Storage. # Set 'https_validate_certificates' to False to disable server certificate # checking. The default for this option in the boto library is currently # 'False' (to avoid breaking apps that depend on invalid certificates); it is # therefore strongly recommended to always set this option explicitly to True # in configuration files, to protect against "man-in-the-middle" attacks. https_validate_certificates = True # 'debug' controls the level of debug messages printed for the XML API only: # 0 for none, 1 for basic boto debug, 2 for all boto debug plus HTTP # requests/responses. #debug = <0, 1, or 2> # 'num_retries' controls the number of retry attempts made when errors occur # during data transfers. The default is 6. # Note 1: You can cause gsutil to retry failures effectively infinitely by # setting this value to a large number (like 10000). Doing that could be useful # in cases where your network connection occasionally fails and is down for an # extended period of time, because when it comes back up gsutil will continue # retrying. However, in general we recommend not setting the value above 10, # because otherwise gsutil could appear to "hang" due to excessive retries # (since unless you run gsutil -D you won't see any logged evidence that gsutil # is retrying). # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing # HTTP connections. #num_retries = <integer value> # 'max_retry_delay' controls the max delay (in seconds) between retries. The # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. # Note: At present this value only impacts the XML API and the JSON API uses a # fixed value of 60. #max_retry_delay = <integer value> aB [GoogleCompute] # 'service_account' specifies the a Google Compute Engine service account to # use for credentials. This value is intended for use only on Google Compute # Engine virtual machines and usually lives in /etc/boto.cfg. Most users # shouldn't need to edit this part of the config. #service_account = default az0 [GSUtil] # 'resumable_threshold' specifies the smallest file size [bytes] for which # resumable Google Cloud Storage uploads are attempted. The default is 8388608 # (8 MiB). #resumable_threshold = %(resumable_threshold)d # 'rsync_buffer_lines' specifies the number of lines of bucket or directory # listings saved in each temp file during sorting. (The complete set is # split across temp files and separately sorted/merged, to avoid needing to # fit everything in memory at once.) If you are trying to synchronize very # large directories/buckets (e.g., containing millions or more objects), # having too small a value here can cause gsutil to run out of open file # handles. If that happens, you can try to increase the number of open file # handles your system allows (e.g., see 'man ulimit' on Linux; see also # http://docs.python.org/2/library/resource.html). If you can't do that (or # if you're already at the upper limit), increasing rsync_buffer_lines will # cause gsutil to use fewer file handles, but at the cost of more memory. With # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes # long, gsutil will require approximately 10 MiB of memory while building # the synchronization state, and will require approximately 60 open file # descriptors to build the synchronization state over all 1M source and 1M # destination URLs. Memory and file descriptors are only consumed while # building the state; once the state is built, it resides in two temp files that # are read and processed incrementally during the actual copy/delete # operations. #rsync_buffer_lines = 32000 # 'state_dir' specifies the base location where files that # need a static location are stored, such as pointers to credentials, # resumable transfer tracker files, and the last software update check. # By default these files are stored in ~/.gsutil #state_dir = <file_path> # gsutil periodically checks whether a new version of the gsutil software is # available. 'software_update_check_period' specifies the number of days # between such checks. The default is 30. Setting the value to 0 disables # periodic software update checks. #software_update_check_period = 30 # 'tab_completion_timeout' controls the timeout (in seconds) for tab # completions that involve remote requests (such as bucket or object names). # If tab completion does not succeed within this timeout, no tab completion # suggestions will be returned. # A value of 0 will disable completions that involve remote requests. #tab_completion_timeout = 5 # 'parallel_process_count' and 'parallel_thread_count' specify the number # of OS processes and Python threads, respectively, to use when executing # operations in parallel. The default settings should work well as configured, # however, to enhance performance for transfers involving large numbers of # files, you may experiment with hand tuning these values to optimize # performance for your particular system configuration. #parallel_process_count = %(parallel_process_count)d #parallel_thread_count = %(parallel_thread_count)d # 'parallel_composite_upload_threshold' specifies the maximum size of a file to # upload in a single stream. Files larger than this threshold will be # partitioned into component parts and uploaded in parallel and then composed # into a single object. # The number of components will be the smaller of # ceil(file_size / parallel_composite_upload_component_size) and # MAX_COMPOSE_ARITY. The current value of MAX_COMPOSE_ARITY is # %(max_compose_arity)d. # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel # uploads will never occur. # Setting an extremely low threshold is unadvisable. The vast majority of # environments will see degraded performance for thresholds below 80M, and it # is almost never advantageous to have a threshold below 20M. # 'parallel_composite_upload_component_size' specifies the ideal size of a # component in bytes, which will act as an upper bound to the size of the # components if ceil(file_size / parallel_composite_upload_component_size) is # less than MAX_COMPOSE_ARITY. # Values can be provided either in bytes or as human-readable values # (e.g., "150M" to represent 150 mebibytes) # # Note: At present parallel composite uploads are disabled by default, because # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), # and for operating systems that don't already have this package installed this # makes gsutil harder to use. Google is actively working with a number of the # Linux distributions to get crcmod included with the stock distribution. Once # that is done we will re-enable parallel composite uploads by default in # gsutil. # # Note: Parallel composite uploads should not be used with NEARLINE, COLDLINE, # or ARCHIVE storage class buckets, as doing this incurs an early deletion # charge for each component object. # # Note: Parallel composite uploads are not enabled with Cloud KMS encrypted # objects as a source or destination, as composition with KMS objects is not yet # supported. #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s #parallel_composite_upload_component_size = %(parallel_composite_upload_component_size)s # # 'parallel_composite_upload_bypass_kms_check' removes the object/bucket KMS checks # used to guard composition of KMS objects. #disable_parallel_composite_upload_kms_check = False # 'sliced_object_download_threshold' and # 'sliced_object_download_component_size' have analogous functionality to # their respective parallel_composite_upload config values. # 'sliced_object_download_max_components' specifies the maximum number of # slices to be used when performing a sliced object download. #sliced_object_download_threshold = %(sliced_object_download_threshold)s #sliced_object_download_component_size = %(sliced_object_download_component_size)s #sliced_object_download_max_components = %(sliced_object_download_max_components)s # Compressed transport encoded uploads buffer chunks of compressed data. When # running a composite upload and/or many uploads in parallel, compression may # consume more memory than available. This setting restricts the number of # compressed transport encoded uploads running in parallel such that they # don't consume more memory than set here. This is 2GiB by default. # Values can be provided either in bytes or as human-readable values # (e.g., "2G" to represent 2 gibibytes) #max_upload_compression_buffer_size = %(max_upload_compression_buffer_size)s # GZIP compression level, if using compression. Reducing this can have # a dramatic impact on compression speed with minor size increases. # This is a value from 0-9, with 9 being max compression. # A good level to try is 6, which is the default used by the gzip tool. #gzip_compression_level = %(gzip_compression_level)s # 'task_estimation_threshold' controls how many files or objects gsutil # processes before it attempts to estimate the total work that will be # performed by the command. Estimation makes extra directory listing or API # list calls and is performed only if multiple processes and/or threads are # used. Estimation can slightly increase cost due to extra # listing calls; to disable it entirely, set this value to 0. #task_estimation_threshold=%(task_estimation_threshold)s # 'use_magicfile' specifies if the 'file --mime <filename>' command should be # used to guess content types instead of the default filename extension-based # mechanism. Available on UNIX and macOS (and possibly on Windows, if you're # running Cygwin or some other package that provides implementations of # UNIX-like commands). When available and enabled use_magicfile should be more # robust because it analyzes file contents in addition to extensions. #use_magicfile = False # Service account emails for testing the hmac command. If these fields are not # populated with distinct service accounts the tests for the hmac command will # not be run. Primarily useful for tool developers. #test_hmac_service_account = #test_hmac_alt_service_account = #test_hmac_list_service_account = # Service account emails for testing impersonation credentials. If this field is # not populated with a service account the tests for service account # impersonation will not run. Primarily useful for tool developers. #test_impersonate_service_account = # 'content_language' specifies the ISO 639-1 language code of the content, to be # passed in the Content-Language header. By default no Content-Language is sent. # See the ISO 639-1 column of # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of # language codes. content_language = en # 'check_hashes' specifies how strictly to require integrity checking for # downloaded data. Legal values are: # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest # will run efficiently (using compiled code), else fail the download. # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a # hash and the local digest computation will run quickly, else skip the # check. # '%(hash_always)s' - Always check download integrity regardless of possible # performance costs. # '%(hash_never)s' - Don't perform download integrity checks. This setting is # not recommended except for special cases such as measuring download # performance excluding time for integrity checking. # This option exists to assist users who wish to download a GCS composite object # and are unable to install crcmod with the C-extension. CRC32c is the only # available integrity check for composite objects, and without the C-extension, # download performance can be significantly degraded by the digest computation. # This option is ignored for daisy-chain copies, which don't compute hashes but # instead (inexpensively) compare the cloud source and destination hashes. #check_hashes = if_fast_else_fail # 'encryption_key' specifies a single customer-supplied encryption key that # will be used for all data written to Google Cloud Storage. See # "gsutil help encryption" for more information # Encryption key: RFC 4648 section 4 base64-encoded AES256 string # Warning: If decrypt_key is specified without an encrypt_key, objects will be # decrypted when copied in the cloud. #encryption_key= # Each 'decryption_key' entry specifies a customer-supplied decryption key that # will be used to access and Google Cloud Storage objects encrypted with # the corresponding key. # Decryption keys: Up to 100 RFC 4648 section 4 base64-encoded AES256 strings # in ascending numerical order, starting with 1. #decryption_key1= #decryption_key2= #decryption_key3= # The ability to specify an alternative JSON API version is primarily for cloud # storage service developers. #json_api_version = v1 # Specifies the API to use when interacting with cloud storage providers. If the # gsutil command supports this API for the provider, it will be used instead of # the default API. Commands typically default to XML for S3 and JSON for GCS. # Note that if any encryption configuration options are set (see above), the # JSON API will be used for interacting with Google Cloud Storage buckets even # if XML is preferred, as gsutil does not currently support this functionality # when using the XML API. #prefer_api = json #prefer_api = xml # Disables the prompt asking for opt-in to data collection for analytics. #disable_analytics_prompt = True # The "test" command runs tests against regional buckets (unless you supply the # `-b` option). By default, the region used is us-central1, but you can change # the default region using this option. #test_cmd_regional_bucket_location = us-central1 # Tests for the "notification watchbucket" command require a notification URL. # If this option is not supplied, those tests will be skipped. #test_notification_url = https://yourdomain.url/notification-endpoint # Used in conjunction with --stet flag on cp command for end-to-end encryption. # STET binary path. If not specified, gsutil checks PATH for "stet". #stet_binary_path = <Path to binary "/usr/local/bin/stet"> # STET config path. If not specified, the STET binary will run with its default # settings. #stet_config_path = ~/.config/my_config.yaml # Adds an API call before parallel operations that triggers a reauth challenge. #trigger_reauth_challenge_for_parallel_operations = False )Zhash_fast_else_failZhash_fast_else_skipZ hash_alwaysZ hash_never�resumable_threshold�parallel_process_count�parallel_thread_count�#parallel_composite_upload_threshold�(parallel_composite_upload_component_size� sliced_object_download_threshold�%sliced_object_download_component_size�%sliced_object_download_max_componentsZmax_compose_arity�task_estimation_threshold�"max_upload_compression_buffer_sizeZgzip_compression_levela0 [OAuth2] # This section specifies options used with OAuth2 authentication. # 'token_cache' specifies how the OAuth2 client should cache access tokens. # Valid values are: # 'in_memory': an in-memory cache is used. This is only useful if the boto # client instance (and with it the OAuth2 plugin instance) persists # across multiple requests. # 'file_system' : access tokens will be cached in the file system, in files # whose names include a key derived from the refresh token the access token # based on. # The default is 'file_system'. #token_cache = file_system #token_cache = in_memory # 'token_cache_path_pattern' specifies a path pattern for token cache files. # This option is only relevant if token_cache = file_system. # The value of this option should be a path, with place-holders '%(key)s' (which # will be replaced with a key derived from the refresh token the cached access # token was based on), and (optionally), %(uid)s (which will be replaced with # the UID of the current user, if available via os.getuid()). # Note that the config parser itself interpolates '%' placeholders, and hence # the above placeholders need to be escaped as '%%(key)s'. # The default value of this option is # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s # where <tmpdir> is the system-dependent default temp directory. # The following options specify the label and endpoint URIs for the OAUth2 # authorization provider being used. Primarily useful for tool developers. #provider_label = Google #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth #provider_token_uri = https://oauth2.googleapis.com/token # 'oauth2_refresh_retries' controls the number of retry attempts made when # rate limiting errors occur for OAuth2 requests to retrieve an access token. # The default value is 6. #oauth2_refresh_retries = <integer value> # The following options specify the OAuth2 client identity and secret that is # used when requesting and using OAuth2 tokens. If not specified, a default # OAuth2 client for the gsutil tool is used; for uses of the boto library (with # OAuth2 authentication plugin) in other client software, it is recommended to # use a tool/client-specific OAuth2 client. For more information on OAuth2, see # http://code.google.com/apis/accounts/docs/OAuth2.html c @ s� e Zd ZdZejdg d�eddddgdddd� Zejdg d �d d e i d �Z d d� Z dd� Z dd� Z dd� Zejdfdd�Zdd� ZdS )� ConfigCommandz(Implementation of gsutil config command.�config)�cfg�conf� configurer z abefhno:rs:w�reauthF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg)r. r/ r0 �aws�s3� command_helpz0Obtain credentials and create configuration file)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s~ t jt jB t jB }tt d�r&|t jO }zt �||d�}W n8 ttfyp } z t d||f ��W Y d}~n d}~0 0 t � |d�S )a Creates and opens a configuration file for writing. The file is created with mode 0600, and attempts to open existing files will fail (the latter is important to prevent symlink attacks). It is the caller's responsibility to close the file. Args: file_path: Path of the file to be created. Returns: A writable file object for the opened file. Raises: CommandException: if an error occurred when opening the file (including when the file already exists). � O_NOINHERITi� z!Failed to open %s for writing: %sN�w) �os�O_RDWR�O_CREAT�O_EXCL�hasattrrD �open�OSError�IOErrorr �fdopen)�self� file_path�flags�fd�e� rT �$/opt/gsutil/gslib/commands/config.py�_OpenConfigFileV s � z ConfigCommand._OpenConfigFilec C s� t jr dS t�|�}ttjtjB |j@ �r�| j� d� t d�}|dv r�z t� |d� | j� d� W q� t y� } z| j� d� W Y d}~q�d}~0 0 n | j� d� dS ) aA Checks that the file has reasonable permissions for a private key. In particular, check that the filename provided by the user is not world- or group-readable. If either of these are true, we issue a warning and offer to fix the permissions. Args: file_path: The name of the private key file. Nz� Your private key file is readable by people other than yourself. This is a security risk, since anyone with this information can use your service account. zDWould you like gsutil to change the file permissions for you? (y/N) )�y�Y� z� The permissions on your file have been successfully modified. The only access allowed is readability by the user (permissions 0400 in chmod).z� We were unable to modify the permissions on your file. If you would like to fix this yourself, consider running: "sudo chmod 400 </path/to/key>" for improved security.z� You have chosen to allow this file to be readable by others. If you would like to fix this yourself, consider running: "sudo chmod 400 </path/to/key>" for improved security.)r � IS_WINDOWSrF �stat�bool�S_IRGRP�S_IROTH�st_mode�logger�warnr �chmod�info� Exception)rO rP �stZfix_it�_rT rT rU � _CheckPrivateKeyFilePermissionss s* � �� �z-ConfigCommand._CheckPrivateKeyFilePermissionsc C s* |sd| }d| }|� d||f � dS )a� Writes proxy name/value pair or comment line to config file. Writes proxy name/value pair if value is not None. Otherwise writes comment line. Args: config_file: File object to which the resulting config file will be written. name: The config variable name. value: The value, or None. desc: Human readable description (for comment). z#%sz<%s>z%s = %s N)�write)rO � config_file�name�value�descrT rT rU � _WriteConfigLineMaybeCommented� s z,ConfigCommand._WriteConfigLineMaybeCommentedc C s� t j}|�d� | �|d|�ddd�d� | �|d|�ddd�d� | �|d|�ddd�d � | �|d |�dd d�d � | �|d |�dd d�d � | �|d|�ddd�d� dS )z�Writes proxy section of configuration file. Args: config_file: File object to which the resulting config file will be written. a? # To use a proxy, edit and uncomment the proxy and proxy_port lines. # If you need a user/password with this proxy, edit and uncomment # those lines as well. If your organization also disallows DNS # lookups by client machines, set proxy_rdns to True (the default). # If you have installed gsutil through the Cloud SDK and have # configured proxy settings in gcloud, those proxy settings will # override any other options (including those set here, along with # any settings in proxy-related environment variables). Otherwise, # if proxy_host and proxy_port are not specified in this file and # one of the OS environment variables http_proxy, https_proxy, or # HTTPS_PROXY is defined, gsutil will use the proxy server specified # in these environment variables, in order of precedence according # to how they are listed above. �proxy�BotoNz proxy host� proxy_typez4proxy type (socks4, socks5, http) | Defaults to http� proxy_portz proxy port� proxy_userz proxy user� proxy_passzproxy password� proxy_rdnsFzLlet proxy server perform DNS lookups (True,False); socks proxy not supported)�botor- rh rm � get_value)rO ri r- rT rT rU � _WriteProxyConfigFileSection� s< � ����� ��z*ConfigCommand._WriteProxyConfigFileSectionTc C s� ddd�}ddd�}i }i }d}|�rJ|t jkr�td�} z@t| d�� } t�| �� � W d � n1 sh0 Y d }W n ty� ts�t d ��Y n0 |s�td �} td � t � d ��d �} | � | � n�|t jkr�t d��nn|t jk�rJd} |D ]J}|dkr�td| �||< td| �||< d } || �r2|| s�t d��q�| �sJt d��|�t�� � |�dtjtj�� �d�f � |�d� |�d� |�r|t jk�r�|�d� |�d| � |�s |�d| � | �s�|�d � |�d � n|�d | � n |�d � nt�� �r|�d � |D ]�}|| }|| }||v �r�||v �r�|�d!||f � |�d"||| f � |�d#||| f � n|�d$||||||f � tj| }|�d%|||f � |dk�r|�d&|||f � |�d'� �q|�t � d(�� |�d)t � | � |� |�t � |�t � |�d*� d+}|t jk�r@d,}|�d-| � t�� �s�td.��!� }d/}|�svt d0��|�d1||f � t"� |�t#� |�d2� dS )3a Creates a boto config file interactively. Needed credentials are obtained interactively, either by asking the user for access key and secret, or by walking the user through the OAuth2 approval flow. Args: config_file: File object to which the resulting config file will be written. cred_type: There are three options: - for HMAC, ask the user for access key and secret - for OAUTH2_USER_ACCOUNT, raise an error - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service account email address and private key file (and if the file is a .p12 file, the password for that file). configure_auth: Boolean, whether or not to configure authentication in the generated file. r; �gs)r; �googler< Fz0What is the full path to your private key file? �rbNTz�Service account authentication via a .p12 file requires either PyOpenSSL or PyCrypto 2.6 or later. Please install either of these to proceed, use a JSON-format key file, or configure a different type of credentials.z,What is your service account email address? � zjWhat is the password for your service key file [if you haven't set one explicitly, leave this line blank]?� z�The user account authentication flow no longer works as of February 1, 2023. Tokens generated before this date will continue to work. To authenticate with your user account, install gsutil via Cloud SDK and run "gcloud auth login"ry z What is your %s access key ID? z#What is your %s secret access key? z2Incomplete credentials provided. Please try again.z*No credentials provided. Please try again.z4# This file was created by gsutil version %s at %s. z%Y-%m-%d %H:%M:%Szl# # You can create additional configuration files by running # gsutil config [options] [-o <config-file>] z[Credentials] z@# Google OAuth2 service account credentials (for "gs://" URIs): zgs_service_key_file = %s zgs_service_client_id = %s a� # If you would like to set your password, you can do so # using the following commands (replaced with your # information): # "openssl pkcs12 -in cert1.p12 -out temp_cert.pem" # "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12" # "rm -f temp_cert.pem" # Your initial password is "notasecret" - for more # information, please see # http://www.openssl.org/docs/apps/pkcs12.html. z!#gs_service_key_file_password = z#gs_service_key_file_password = %s z�# To add Google OAuth2 credentials ("gs://" URIs), edit and uncomment the # following line: #gs_oauth2_refresh_token = <your OAuth2 refresh token> zg# Google OAuth2 credentials are managed by the Cloud SDK and # do not need to be present in this file. z!# %s credentials ("%s://" URIs): z%s_access_key_id = %s z%s_secret_access_key = %s z�# To add HMAC %s credentials for "%s://" URIs, edit and uncomment the # following two lines: #%s_access_key_id = <your %s access key ID> #%s_secret_access_key = <your %s secret access key> a� # The ability to specify an alternate storage host and port # is primarily for cloud storage service developers. # Setting a non-default gs_host only works if prefer_api=xml. #%s_host = <alternate storage host address> #%s_port = <alternate storage host port> # In some cases, (e.g. VPC requests) the "host" HTTP header should # be different than the host used in the request URL. #%s_host_header = <alternate storage host header> z�#%s_json_host = <alternate JSON API storage host address> #%s_json_port = <alternate JSON API storage host port> #%s_json_host_header = <alternate JSON API storage host header> z�# To impersonate a service account for "%s://" URIs over # JSON API, edit and uncomment the following line: #%s_impersonate_service_account = <service account email> a� # This configuration setting enables or disables mutual TLS # authentication. The default value for this setting is "false". When # set to "true", gsutil uses the configured client certificate as # transport credential to access the APIs. The use of mTLS ensures that # the access originates from a trusted enterprise device. When enabled, # the client certificate is auto discovered using the endpoint # verification agent. When set to "true" but no client certificate or # key is found, users receive an error. #use_client_certificate = False # The command line to execute, which prints the # certificate, private key, or password to use in # conjunction with "use_client_certificate = True". #cert_provider_command = <Absolute path to command to run for # certification. Ex: "/scripts/gen_cert.sh"> z%s z� # 'default_api_version' specifies the default Google Cloud Storage XML API # version to use. If not set below gsutil defaults to API version 1. � r zdefault_api_version = %d zWhat is your project-id? z� # 'default_project_id' specifies the default Google Cloud Storage project ID to # use with the 'mb' and 'ls' commands. This default can be overridden by # specifying the -p option to the 'mb' and 'ls' commands. zlNo default project ID entered. The default project ID is needed by the ls and mb commands; please try again.z %sdefault_project_id = %s zH#client_id = <OAuth2 client id> #client_secret = <OAuth2 client secret> )$r �OAUTH2_SERVICE_ACCOUNTr rK �json�loads�read� ValueErrorr r �join�textwrap�wraprg �OAUTH2_USER_ACCOUNT�HMACrh �CONFIG_PRELUDE_CONTENT�lstrip�gslib�VERSION�datetime�now�strftimer �InvokedViaCloudSdkr � HostKeyMap�dedent�CONFIG_BOTO_SECTION_CONTENTrw �$CONFIG_GOOGLECOMPUTE_SECTION_CONTENT�'CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT�stripr � CONFIG_OAUTH2_CONFIG_CONTENT)rO ri � cred_type�configure_authZ provider_mapZuri_mapZkey_idsZsec_keysZservice_account_key_is_json�gs_service_key_fileZ key_file_fp�gs_service_client_id� gs_service_key_file_passwordZ got_creds�provider� key_prefixZ uri_schemeZhost_key� api_version�default_project_idZproject_id_section_preluderT rT rU �_WriteBotoConfigFile� s , � ��� � � � ���� �� �� �� � ��� �� ���� �� z"ConfigCommand._WriteBotoConfigFilec C sF t j}d}d}d}d}| jD ]T\}}|dkr8t j}d}q |dkrLt j}d}q |dkrZd}q |dkrh|}q | �� q |r�|r�td��|s�|s�|r�td ��t�� r�t� � r�|s�|r�td � d d d � t j d ddd��d � t j dddd��g���t�� �r|�rt j�d � t � d��d � d}|�s�tj�dd�} | �r8| } ntj�tj� dd��} tj�| ��sb| }n�| d }tj�|��r�td| |f ��nbz$t j�d| |f � t�| |� W n8 t�y� } z td| || f ��W Y d} ~ n d} ~ 0 0 | }|dk�r�t j} n*| �|�} t j�d � t � d | ��d � ttj t � z| j | ||d � W n� t�y� } z�t | t �} | �rvt j�d � |dk�r�| �!� t�"|� z&|�r�t j�d | � t�||� W n* t�y� } z| �W Y d} ~ n d} ~ 0 0 � W Y d} ~ n d} ~ 0 0 |dk�rB| �!� t#j$�%d d!��sBt j�d d � t � d"| �� d � d#S )$z+Command entry point for the config command.NFTz-az-ez-nz-ozYBoth -a and -e cannot be specified. Please see "gsutil help config" for more information.zoThe -a and -e flags cannot be specified with the -n flag. Please see "gsutil help config" for more information.r{ zDOAuth2 is the preferred authentication mechanism with the Cloud SDK.z<Run "gcloud auth login" to configure authentication, unless:z�You don't want gsutil to use OAuth2 credentials from the Cloud SDK, but instead want to manage credentials with .boto files generated by running "gsutil config"; in which case run "gcloud config set pass_credentials_to_gsutil false".z- z )�initial_indent�subsequent_indentzbYou want to authenticate with an HMAC access key and secret, in which case run "gsutil config -a".z�This command will configure HMAC credentials, but gsutil will use OAuth2 credentials from the Cloud SDK by default. To make sure the HMAC credentials are used, run: "gcloud config set pass_credentials_to_gsutil false".z Z BOTO_CONFIG�~z.botoz.bakzDCannot back up existing config file "%s": backup file exists ("%s").z0Backing up existing config file "%s" to "%s"... z:Failed to back up existing config file ("%s" -> "%s"): %s.�-z�This command will create a boto config file at %s containing your credentials, based on your responses to the following questions.)r� r� z Caught ^C; cleaning up z'Restoring previous backed up file (%s) ro rn zzBoto config file "%s" created. If you need to use a proxy to access the Internet please see the instructions in that file.r )&r r� �sub_optsr� r~ � RaiseInvalidArgumentExceptionr r r� �CloudSdkCredPassingEnabledr� r� r� �sys�stderrrh rF �environ�get�path� expanduser�exists�renamerd �stdoutrV r �signal�SIGINT�_CleanupHandlerr� � isinstancer �close�unlinkru r- � has_option)rO r� Zoutput_file_nameZhas_aZhas_er� �optZopt_argZdefault_config_path_bakZboto_config_from_envZdefault_config_pathrS � output_fileZ user_abortedrT rT rU � RunCommand� s ��������� �������� ���� � � ����zConfigCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrV rg rm rw r r� r� r� rT rT rT rU r, ; s< � � ,.� dr, c C s t d��d S )Nz User interrupted config commandr )Zunused_signalnumZunused_handlerrT rT rU r� R s r� )Mr� � __future__r r r r r� r �multiprocessingrF r� �socketr[ r� r� �time� webbrowser� six.movesr �six.moves.http_clientr ru � boto.providerr r� � gslib.commandr r Zgslib.commands.composer �gslib.cred_typesr �gslib.exceptionr r Z gslib.metricsr �gslib.sig_handlingr Z gslib.utilsr r �gslib.utils.hashing_helperr r r r �&gslib.utils.parallelism_framework_utilr �httplib2r �oauth2client.clientr r� r� ZGOOG_CLOUD_CONSOLE_URIr� � should_prohibit_multiprocessingZ unused_os� DEFAULT_PARALLEL_PROCESS_COUNT� DEFAULT_PARALLEL_THREAD_COUNT�min� cpu_countZ+DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLDZ0DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZEZ(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLDZ-DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZEZ-DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTSZ*DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZEZ DEFAULT_GZIP_COMPRESSION_LEVELr� r� �RESUMABLE_THRESHOLD_Br� r� r, r� rT rT rT rU �<module> s� ��] 0 k�� � 0 __pycache__/signurl.cpython-39.pyc 0000644 00000051764 15025166704 0013113 0 ustar 00 a +(Wg�s � @ sx d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z dd lm Z ddl Z ddl Z ddl Z ddlZddlZdd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl!m"Z" ddl!m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. z8ddl/m0Z0 ddl/m1Z1 ddl/m2Z2 dd l/m3Z3 d Z4W n$ e5�y� dZ1dZ2d Z4dZ0Y n0 zDdd l6m7Z7 dd l8m9Z9 dd!l:m;Z; dd"l<m=Z= dd#l>m?Z? d Z@W n e5�y d Z@d$ZAY n0 d%ZBe d&d'�ZCe d(d)�ZDd*ZEd+eE d, ZFd-d.� ZGd/d0� ZHd9d1d2�ZId3d4� ZJd:d5d6�ZKG d7d8� d8e�ZLdS );zpImplementation of URL Signing workflow. see: https://cloud.google.com/storage/docs/access-control#Signed-URLs) � )�absolute_import)�print_function)�division)�unicode_literalsN)�datetime)� timedelta)�timezone)�urllib)� HttpError)� MakeRequest)�Request)�config)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)� constants)� GetNewHttp)�GcloudStorageMap�GcloudStorageFlag)� CreatePayload� GetFinalUrl�to_bytes)� FILETYPE_PEM)�load_privatekey)�sign)�PKeyTF)�hashes)�padding)� RSAPrivateKey)�pkcs12)�NameOIDzapyca/cryptography is not available. Either install it, or please consider using the .json keyfile�auto� ��days� ��hoursz� gsutil signurl [-c <content_type>] [-d <duration>] [-m <http_method>] \ [-p <password>] [-r <region>] [-b <project>] (-u | <private-key-file>) \ (gs://<bucket_name> | gs://<bucket_name>/<object_name>)... z <B>SYNOPSIS</B> aG <B>DESCRIPTION</B> The signurl command will generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. Please see the `Signed URLs documentation <https://cloud.google.com/storage/docs/access-control/signed-urls>`_ for background about signed URLs. Multiple gs:// URLs may be provided and may contain wildcards. A signed URL will be produced for each provided URL, authorized for the specified HTTP method and valid for the given duration. NOTE: Unlike the gsutil ls command, the signurl command does not support operations on sub-directories. For example, unless you have an object named ``some-directory/`` stored inside the bucket ``some-bucket``, the following command returns an error: ``gsutil signurl <private-key-file> gs://some-bucket/some-directory/`` The signurl command uses the private key for a service account (the '<private-key-file>' argument) to generate the cryptographic signature for the generated URL. The private key file must be in PKCS12 or JSON format. If the private key is encrypted the signed URL command will prompt for the passphrase used to protect the private key file (default 'notasecret'). For more information regarding generating a private key for use with the signurl command please see the `Authentication documentation. <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_ If you used `service account credentials <https://cloud.google.com/storage/docs/gsutil/addlhelp/CredentialTypesSupportingVariousUseCases#supported-credential-types_1>`_ for authentication, you can replace the <private-key-file> argument with the -u or --use-service-account option to use the system-managed private key directly. This avoids the need to store a private key file locally, but prior to using this flag you must `configure <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ ``gcloud`` to use your service account credentials. <B>OPTIONS</B> -b <project> Allows you to specify a user project that will be billed for requests that use the signed URL. This is useful for generating presigned links for buckets that use requester pays. Note that it's not valid to specify both the ``-b`` and ``--use-service-account`` options together. -c Specifies the content type for which the signed URL is valid for. -d Specifies the duration that the signed URL should be valid for, default duration is 1 hour. Times may be specified with no suffix (default hours), or with s = seconds, m = minutes, h = hours, d = days. This option may be specified multiple times, in which case the duration the link remains valid is the sum of all the duration options. The max duration allowed is 7 days when ``private-key-file`` is used. The max duration allowed is 12 hours when -u option is used. This limitation exists because the system-managed key used to sign the URL may not remain valid after 12 hours. -m Specifies the HTTP method to be authorized for use with the signed URL, default is GET. You may also specify RESUMABLE to create a signed resumable upload start URL. When using a signed URL to start a resumable upload session, you will need to specify the 'x-goog-resumable:start' header in the request or else signature validation will fail. -p Specify the private key password instead of prompting. -r <region> Specifies the `region <https://cloud.google.com/storage/docs/locations>`_ in which the resources for which you are creating signed URLs are stored. Default value is 'auto' which will cause gsutil to fetch the region for the resource. When auto-detecting the region, the current gsutil user's credentials, not the credentials from the private-key-file, are used to fetch the bucket's metadata. This option must be specified and not 'auto' when generating a signed URL to create a bucket. -u Use service account credentials instead of a private key file to sign the URL. You can also use the ``--use-service-account`` option, which is equivalent to ``-u``. Note that both options have a maximum allowed duration of 12 hours for a valid link. <B>USAGE</B> Create a signed URL for downloading an object valid for 10 minutes: gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object> Create a signed URL, valid for one hour, for uploading a plain text file via HTTP PUT: gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to PUT to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to POST a resumable upload to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m RESUMABLE -d 1d -c image/jpg <private-key-file> \ gs://bucket/<obj> c C s t jtjd�jdd�S )z2Returns the current utc time as a datetime object.��tzN��tzinfo)r �nowr �utc�replace� r2 r2 �%/opt/gsutil/gslib/commands/signurl.py�_NowUTC� s r4 c C s� t �d| �}|std��|�d�\} }t| �} |�� }|dkrJt| d�}n:|dkr^t| d�}n&|dkrrt| d�}n|d kr�t| d �}|S ) z>Parses the given duration and returns an equivalent timedelta.z^(\d+)([dDhHmMsS])?$z Unable to parse duration string�h�dr& r) �m)�minutes�s��seconds)�re�matchr �groups�int�lowerr )�durationr= �modifier�retr2 r2 r3 �_DurationToTimeDelta� s rD c C s� t �ddd�}d|i}|dkr8d}d|d< | s8|�d � | rD| |d < |rf|j||||| || || d � }n|tjrrd }nd }t||||| || || | d� \}}t| t�r�t | ||�}n&t s�t t ��| � t |�t�� t�� �}t||||�}|S )a1 Construct a string to sign with the provided key. Args: key: The private key to use for signing the URL. api: The CloudApiDelegator instance use_service_account: If True, use the service account credentials instead of using the key file to sign the URL provider: Cloud storage provider to connect to. If not present, class-wide default is used. client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. gcs_path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. content_type: Optional Content-Type for the signed URL. HTTP requests using the URL must match this Content-Type. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. generation: If not None, specifies a version of an object for signing. Returns: The complete URL (string). � Credentials�gs_hostzstorage.googleapis.com�host� RESUMABLE�POST�startzx-goog-resumablez�Warning: no Content-Type header was specified with the -c flag, so uploads to the resulting Signed URL must not specify a Content-Type.z content-type) �provider�methodrA �path� generation�logger�region�signed_headers�string_to_sign_debugs RSA-SHA256z RSA-SHA256) � client_idrL rA rM rN rO rP rQ �billing_projectrR )r �get�warn�SignUrl�six�PY2r � isinstancer r � HAVE_CRYPTOr �_CRYPTO_IMPORT_ERRORr r �PKCS1v15r �SHA256r )�key�api�use_service_accountrK rS rL rA �gcs_pathrO rP � content_typerT rR rN rF rQ � final_url�digest�string_to_sign�canonical_query_string� raw_signaturer2 r2 r3 � _GenSignedUrl� sX ) � � �ri c C sf t | �} t |�}ts tt��z0t�| |�\}}}|j�tj �d j }||fW S td��Y n0 d S )Nr z<Unable to load the keyfile, Invalid password or PKCS12 data.) r r[ r r\ r" �load_key_and_certificates�subject�get_attributes_for_oidr# � COMMON_NAME�value)� key_string�passwdr_ �cert�add� client_emailr2 r2 r3 � _ReadKeystoreR s rt c C s\ t �t�| ��}d|vs d|vr(td��|d }|rFtt|d |�}ntt|d �}||fS )a; Read the client email and private key from a JSON keystore. Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console aren't encrypted so the passwd is optional as load_privatekey will prompt for the PEM passphrase if the key is encrypted. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must be a valid JSON string and contain the fields 'private_key' and 'client_email'. passwd: Passphrase for encrypted private keys. Returns: key: Parsed private key from the keystore. client_email: The email address for the service account. Raises: ValueError: If unable to parse ks_contents or keystore is missing required fields. rs � private_keyz-JSON keystore doesn't contain required fields)�json�loadsrX � ensure_str� ValueErrorr r )Z ks_contentsrp �ksrs r_ r2 r2 r3 �_ReadJSONKeystore_ s r{ c s� e Zd ZdZejdddgedejddgddde j e j ge j e � � e �� gd � Zejdddgd d ei d �Z� fd d�Zdd� Zdd� Zdd� Zdd� Z� ZS )�UrlSignCommandz*Implementation of gsutil url_sign command.ZsignurlZ signedurlZ queryauth� z m:d:b:c:p:r:uzuse-service-accountF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzCreate a signed URL)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t �| j�}t �| j�}dddd| jd g}| jdd � | _d }d }d }d }t| j�D ]B\}\} } | dkrr|}qX| dkr�|}qX| d kr�|}qX| d krX|}qX|d ur�ttt| j| d ��� ��d } d| f| j|< |d u�r| j| d d k�rd | j|< |dg7 }|d u�r0| j| d } d d| f| j|< |d u�rZ| j| d } d d| f| j|< t � � t |t d�t d�t d�t d�t d�t d�d�d��}|| _|| _|S )N�storagezsign-urlz�--format=csv[separator="\t"](resource:label="URL", http_verb:label="HTTP Method", expiration:label="Expiration", signed_url:label="Signed URL")z--private-key-file=r r} �-d�-m�-c�-br9 rH )r� rI z --headers=x-goog-resumable=startz content-type=z userProject=z --http-verbz --durationz--query-paramsz --headersz--regionz--private-key-password)r� r� r� r� �-r�-p)�gcloud_command�flag_map) �copy�deepcopy�args�sub_opts� enumerate�strr? rD � total_seconds�super�get_gcloud_storage_argsr r )�selfZ original_argsZoriginal_sub_optsr� Zduration_arg_idxZhttp_verb_arg_idxZcontent_type_arg_idxZbilling_project_arg_idx�i�flag�_r; Zcontent_type_valueZ project_valueZfully_translated_command�� __class__r2 r3 r� � sv � ���� � � ���z&UrlSignCommand.get_gcloud_storage_argsc C sp d }d}d}d }t }d}d }| jD ]�\}} tjrD| �tjjp@tj �} |dkrl|d urb|t | �7 }q�t | �}q"|dkrz| }q"|dkr�| }q"|dkr�| }q"|dkr�| }q"|d ks�|d kr�d }q"|d kr�| }q"| � � q"|d u r�t d d�}n4|�r|t k�rtdt ��n|tk�rtdt ��|dv�r,td��|�sJt| j�dk �rJtd��|�r^|�r^td��|||||||fS )N�GET� Fr� r� r� r� r� z-uz--use-service-accountTr� r} r) z�Max valid duration allowed is %s when -u flag is used. For longer duration, consider using the private-key-file instead of the -u option.z Max valid duration allowed is %s)r� �PUT�DELETE�HEADrH z9HTTP method must be one of[GET|HEAD|PUT|DELETE|RESUMABLE]� z�The command requires a key file argument and one or more URL arguments if the --use-service-account flag is missing. Run `gsutil help signurl` for more infozLSpecifying both the -b and --use-service-account options together isinvalid.)�_AUTO_DETECT_REGIONr� rX rY �decode�sys�stdin�encodingr �UTF8rD � RaiseInvalidArgumentExceptionr �!_MAX_EXPIRATION_TIME_WITH_MINUS_Ur �_MAX_EXPIRATION_TIME�lenr� ) r� �deltarL rc rp rP ra rT �o�vr2 r2 r3 �_ParseAndCheckSubOpts� sd �� � � �z$UrlSignCommand._ParseAndCheckSubOptsc C s� t || j|||dtdd�||||| dd� } z6t� } t| d�} t| | �} | jdvrZt�| ��| jW S ty� } zL|� d�r�|j }d|j }|j r�|d |j 7 }nd | }t |��W Y d }~n d }~0 0 d S ) zFPerforms a head request against a signed URL to check for read access.r� �< r: T) r_ r` ra rK rS rL rA rb rN rO rP rT rR )�� � � �responsezbUnexpected HTTP response code %s while querying object readability. Is your system clock accurate?z Content: %szbExpected an HTTP response code of 200 while querying object readability, but received an error: %sN) ri � gsutil_apir r r r � status_coder � FromResponseZhas_attrr� �contentr )r� r_ ra rK rs rb rN rO rP rT Z signed_urlr5 �reqr� � http_errorZerror_responseZ error_stringr2 r2 r3 � _ProbeObjectAccessWithClient1 sB � ��z+UrlSignCommand._ProbeObjectAccessWithClientc C sD g }|D ]6}t |�r0|�dd� | �|�D �� q|�t|�� q|S )Nc S s g | ] }|j �qS r2 )� storage_url)�.0�blrr2 r2 r3 � <listcomp>b � z8UrlSignCommand._EnumerateStorageUrls.<locals>.<listcomp>)r �extend�WildcardIterator�appendr )r� Zin_urlsrC �url_strr2 r2 r3 �_EnumerateStorageUrls] s z$UrlSignCommand._EnumerateStorageUrlsc C sB t s td��| �� \}}}}}}}|r*dnd}| �| j|d� �} i } d} |s�z"tt| jd d��� |�\} } W q� ty� |s�t � d�}z"t t| jd d��� |�\} } W n& ty� td� | jd ���Y n0 Y q�0 n| j j dd �} td � | D �]B} | jdk�rtd ��| �� �rH|tk�r.td ��| j}|d k�rltd��n$d� | jtjj| j�tj�dd��}|tk�r| j| v �r�| | j }nrz | jd� | j�dgd�\}}W n< t�y� } z"td� |j j | j���W Y d}~n d}~0 0 |j � � }|| | j< n|}t | | j || j| |||| j!| j"|||dd�}t#�$t%j&t'j(d�j)dd�| �*� �}t%�+|�}|�,d�}t-j.�r~|�/tj�}d� | j0|||�}t-j.�r�|�tj�}t|� | �1| || j| || j!| j"||� }|d k�r | �� �r�|d k�r�td � | ���n"|d k�r<|d k�r<td � | ���q�|d kr�| j"�2d!| �p6d"| � q�dS )#z(Command entry point for signurl command.zhThe signurl command requires the pyopenssl library (try pip install pyopenssl or easy_install pyopenssl)r r} N�rbzKeystore password:z$Unable to parse private key from {0}�gs)rK z%URL HTTP Method Expiration Signed URLz+Can only create signed urls from gs:// urlsz�Generating signed URLs for creating buckets requires a region be specified via the -r option. Run `gsutil help signurl` for more information about the '-r' option.rH z-Resumable signed URLs require an object name.z{0}/{1}s /~)�safezgs://{}�location)� bucket_fieldsz�{}: Failed to auto-detect location for bucket '{}'. Please ensure you have storage.buckets.get permission on the bucket or specify the bucket's location using the '-r' option.T)r_ r` ra rK rS rL rA rb rN rO rP rc rT rR r+ r- z%Y-%m-%d %H:%M:%Sz{0} {1} {2} {3}r� r� zkBucket {0} does not exist. Please create a bucket with that name before a creating signed URL to access it.zsObject {0} does not exist. Please create/upload an object with that name before a creating signed URL to access it.r� z�%s does not have permissions on %s, using this link will likely result in a 403 error until at least READ permissions are grantedz The account)3� HAVE_OPENSSLr r� r� r� r{ �open�readry �getpassrt �formatr� �GetServiceAccountId�print�scheme�IsBucketr� � bucket_namer �parse�quote� object_name�encoder r� �GetSingleBucketUrlFromArg� Exceptionr� �__name__r� r@ ri rN rO �calendar�timegmr r/ r r0 r1 � utctimetuple� fromtimestamp�strftimerX rY r� � url_stringr� rV )r� rL r� rc rp rP ra rT Zarg_start_indexZ storage_urlsZ region_cacher_ rs �urlrb Z bucket_regionr� �bucket�erd � expirationZ expiration_dtZtime_strZ url_info_str� response_coder2 r2 r3 � RunCommandh s� �� � � � �� � �� �$ � � ���� �zUrlSignCommand.RunCommand)r� � __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreFileURLsArgument� MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� � __classcell__r2 r2 r� r3 r| � sB ���� GB, r| )NNFN)N)Mr� � __future__r r r r r� r� r r r r� rv r< r� rX � six.movesr Zapitools.base.py.exceptionsr Z apitools.base.py.http_wrapperr r �botor � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r Z gslib.utilsr Zgslib.utils.boto_utilr �gslib.utils.shim_utilr r �gslib.utils.signurl_helperr r r ZOpenSSL.cryptor r r r r� � ImportError� cryptography.hazmat.primitivesr �)cryptography.hazmat.primitives.asymmetricr Z-cryptography.hazmat.primitives.asymmetric.rsar! �,cryptography.hazmat.primitives.serializationr" �cryptography.hazmat._oidr# r[ r\ r� r� r� r� r r4 rD ri rt r{ r| r2 r2 r2 r3 �<module> s� ��! � ^ & __pycache__/hash.cpython-39.pyc 0000644 00000017577 15025166704 0012357 0 ustar 00 a +(WgD' � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl mZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl m!Z! ddl m"Z" ddlm#Z# e j$Z%dZ&de& d Z'de#�(� d d e#�(� d d d d d! e#�)� d" d# d$ d% d! e#�)� d& d' Z*G d(d)� d)e �Z+dS )*zEImplementation of hash command for calculating hashes of local files.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)�CommandArgument)� ApiSelector)�CommandException)�FileProgressCallbackHandler)�ProgressCallbackWithTimeout)�StorageUrlFromString)� FileMessage)� FinalMessage)� boto_util)� constants)�hashing_helper)�parallelism_framework_util)�GcloudStorageFlag)�GcloudStorageMap)� shim_utilz* gsutil hash [-c] [-h] [-m] filename... z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> Calculate hashes on local files, which can be used to compare with ``gsutil ls -L`` output. If a specific hash option is not provided, this command calculates all gsutil-supported hashes for the files. Note that gsutil automatically performs hash validation when uploading or downloading files, so this command is only needed if you want to write a script that separately checks the hash. If you calculate a CRC32c hash for files without a precompiled crcmod installation, hashing will be very slow. See "gsutil help crcmod" for details. <B>OPTIONS</B> -c Calculate a CRC32c hash for the specified files. -h Output hashes in hex format. By default, gsutil uses base64. -m Calculate a MD5 hash for the specified files. z>--format=value[separator="",terminator=""](digest_format.sub("z", "Hashes ["),z url.sub("z ", "] for ").sub("$", ":^\^n"),z,md5_hash.yesno(yes=" Hash (md5): ", no=""),zmd5_hash.yesno(no=""),z$md5_hash.yesno(yes="NEWLINE", no="")z.sub("NEWLINE", "z"),z2crc32c_hash.yesno(yes=" Hash (crc32c): ", no=""),zcrc32c_hash.yesno(no=""),z'crc32c_hash.yesno(yes="NEWLINE", no="")z")�)c s� e Zd ZdZejdg edejdddde j ge j e � � gd� Z ejdd gd d ei d �Z� fd d�Zedd� �Zdd� Zdd� Z� ZS )� HashCommandz&Implementation of gsutil hash command.�hash� ZchmTFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZchecksum� command_helpzCalculate file hashes)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t ddtgtd�d d d�d�}t| jdd� | jD � �}d|v rZd |vrZ| jd g7 _n d |v rzd|vrz| jd g7 _t� �|�S ) N�storager z--hex)�-h�-c�-m)�gcloud_command�flag_mapc S s g | ] \}}|�qS � r2 )�.0�flag�_r2 r2 �"/opt/gsutil/gslib/commands/hash.py� <listcomp>� � z7HashCommand.get_gcloud_storage_args.<locals>.<listcomp>r. r/ z --skip-md5z --skip-crc32c) r �_GCLOUD_FORMAT_STRINGr �set�args�sub_optsr0 �super�get_gcloud_storage_args)�self�gcloud_storage_mapZargs_set�� __class__r2 r6 r> t s ��� z#HashCommand.get_gcloud_storage_argsc C s� d}d}dd� }dd� }d}d}|rr|D ]H\} } | dkrBd}d}q(| dkr`d }d d� }d d� }q(| d kr(d}d}q(|s~d}d}|r�t �� s�|�tj� |||||fS ) aZ Returns behavior variables based on input options. Args: sub_opts: getopt sub-arguments for the command. logger: logging.Logger for the command. Returns: Tuple of calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum. calc_md5: Boolean, if True, command should calculate an MD5 hash. format_func: Function used for formatting the hash in the desired format. cloud_format_func: Function used for formatting the hash in the desired format. output_format: String describing the hash output format. Fc S s t �| �� �S �N)r �Base64EncodeHash� hexdigest��digestr2 r2 r6 �<lambda>� r8 z(HashCommand._ParseOpts.<locals>.<lambda>c S s | S rC r2 rF r2 r2 r6 rH � r8 �base64r. Tr- �hexc S s | � � S rC )rE rF r2 r2 r6 rH � r8 c S s t �| ��d�S )N�ascii)r �Base64ToHexHash�decoderF r2 r2 r6 rH � s r/ )r �UsingCrcmodExtension�warnr �SLOW_CRCMOD_WARNING) �clsr< �logger� calc_crc32c�calc_md5� format_func�cloud_format_funcZfound_hash_option� output_format�o�unused_ar2 r2 r6 � _ParseOpts� s2 � zHashCommand._ParseOptsc C s, i }|rt j�d�|d< |r(t�� |d< |S )a4 Constructs the dictionary of hashes to compute based on the arguments. Args: calc_crc32c: If True, CRC32c should be included. calc_md5: If True, MD5 should be included. Returns: Dictionary of {string: hash digester}, where string the name of the digester algorithm. zcrc-32c�crc32c�md5)�crcmod� predefined�Crcr �GetMd5)r? rS rT � hash_dictr2 r2 r6 �_GetHashClassesFromArgs� s z#HashCommand._GetHashClassesFromArgsc C s� | � | j| j�\}}}}}d}| jD �]�}| �|�jg d�d�D �]�}d}t|�} |jj} t|�� � �r"t j � | �} | j j�t| dt�� | dtjd�� t| t| j jt|�dd�j�} | �||�} t| d �� }tj|| | d � W d � n1 s�0 Y | j j�t| dt�� | dtjd�� nbi } |j}|j} |j du}|j du}|�sd|�sdt � � � d |� q@|�rt|j | d < |�r�|j | d < t!d|| f � t"�#| �D ]0\}}t!d|| � � �r�||�n||�f � �q�q@q$|�s�t$d��t%| j jt&t�� �� dS )z)Command entry point for the hash command.F)r[ �customerEncryption�md5Hash�size)�bucket_listing_fieldsTN)re �finished� message_type�Hashing)�src_url�operation_name�rb)�callback_processorzNo hashes present for %sr\ r[ zHashes [%s] for %s:z Hash (%s): %szNo files matchedr )'rZ r< rR r; �WildcardIterator� IterObjectsr � storage_url� object_name� IsFileUrl�os�path�getsize� gsutil_api� status_queue�putr �time� FILE_HASHr r �callrb �openr �CalculateHashesFromContents� root_objectre rd r[ �logging� getLoggerrO �print�six� iteritemsr �_PutToQueueWithTimeoutr )r? rS rT rU rV rW Z matched_one�url_strZfile_ref�url� file_name� file_sizerm ra �fp� obj_metadataZ md5_presentZcrc32c_present�namerG r2 r2 r6 � RunCommand� s� � � ���� �$�� � �zHashCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �JSONr � MakeZeroOrMoreFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr> � classmethodrZ rb r� � __classcell__r2 r2 rA r6 r Y s8 �� 0r ),r� � __future__r r r r r rs ry r] r� � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.progress_callbackr r �gslib.storage_urlr �gslib.thread_messager r Z gslib.utilsr r r r �gslib.utils.shim_utilr r r �PutToQueueWithTimeoutr� r� r� �get_format_flag_caret�get_format_flag_newliner9 r r2 r2 r2 r6 �<module> s� ��������������������� __pycache__/defstorageclass.cpython-39.pyc 0000644 00000013437 15025166704 0014574 0 ustar 00 a +(Wg� � @ s^ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z dZ dZ e e � d� Z!dZ"dZ#dd�$e"e# g� Z%ee!e%�Z&ee e#�Z'ee e"�Z(de �)� d Z*eddd e*d gi d �Z+eg d �i d �Z,G d d!� d!e �Z-d"S )#zGImpl. of default bucket storage class command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�NormalizeStorageClass)� shim_utilzD gsutil defstorageclass set <storage-class> gs://<bucket_name>... z4 gsutil defstorageclass get gs://<bucket_name>... � a{ <B>SET</B> The "defstorageclass set" command sets the default `storage class <https://cloud.google.com/storage/docs/storage-classes>`_ for the specified bucket(s). If you specify a default storage class for a certain bucket, Google Cloud Storage applies the default storage class to all new objects uploaded to that bucket, except when the storage class is overridden by individual upload requests. Setting a default storage class on a bucket provides a convenient way to ensure newly uploaded objects have a specific storage class. If you don't set the bucket's default storage class, it will default to Standard. z; <B>GET</B> Gets the default storage class for a bucket. z5 The defstorageclass command has two sub-commands: z)--format=value[separator=": "](name.sub("z", "gs://"),storageClass)�storage�buckets�listz--raw��gcloud_command�flag_map)r r �updatez--default-storage-classc @ s� e Zd ZdZejdededdddej ej gej e � � e � � ge � � gd�d� Zejddgd d eeed �d �Zeeed �i d �Zdd� Zdd� Zdd� Zdd� Zdd� Z dS )�DefStorageClassCommandz1Implementation of gsutil defstorageclass command.Zdefstorageclass� � F)�set�get) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZdefaultstorageclass� command_helpz/Get or set the default storage class on buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr c C s |� d�s td| j|f ��d S )Nzgs://zD"%s" does not support the URL "%s". Did you mean to use a gs:// URL?)� startswithr � command_name)�self�url_str� r4 �-/opt/gsutil/gslib/commands/defstorageclass.py� _CheckIsGsUrl� s ��z$DefStorageClassCommand._CheckIsGsUrlc C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)r2 r4 r4 r5 �_CalculateUrlsStartArg� s z-DefStorageClassCommand._CalculateUrlsStartArgc C s� t | jd �}| jdd� }|s(| �� d}|D ]t}| �|� | j|dgd�}|D ]P}d}t�� }| j�d||j � d �f � ||_ | j j |jj||jjdgd � qRq0|s�ttt|� ��dS ) z,Sets the default storage class for a bucket.r r7 NF�id�� bucket_fieldsTz3Setting default storage class to "%s" for bucket %s�/)�provider�fields)r r8 r9 r6 �GetBucketUrlIterFromArg�apitools_messages�Bucket�logger�info� url_string�rstrip� storageClass� gsutil_api� PatchBucket� storage_url� bucket_name�schemer r r )r2 �normalized_storage_class�url_args� some_matchedr3 � bucket_iter�blr�bucket_metadatar4 r4 r5 �_SetDefStorageClass� s, � � z*DefStorageClassCommand._SetDefStorageClassc C sp | j }d}|D ]H}| �|� | j|dgd�}|D ]$}d}td|j�d�|jjf � q0q|sltt t |� ��dS )z,Gets the default storage class for a bucket.FrI r= Tz%s: %sr? N) r8 r6 rB �printrG rH � root_objectrI r r r )r2 rP rQ r3 rR rS r4 r4 r5 �_GetDefStorageClass� s �� z*DefStorageClassCommand._GetDefStorageClassc C sv | j �d�}|g}|dkr"| j}n>|dkrJ| j}t| j d �}|�|� ntd|| j| jf ��tj |d� |� dS )z4Command entry point for the defstorageclass command.r r r zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".)� subcommands) r8 �poprX rU r �appendr r1 r �LogCommandParams)r2 �action_subcommand�subcommand_args�funcrO r4 r4 r5 � RunCommand� s �� z!DefStorageClassCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �MakeFreeTextArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �SHIM_GET_COMMAND_MAP�SHIM_SET_COMMAND_MAP�gcloud_storage_mapr6 r; rU rX r` r4 r4 r4 r5 r \ sN ����� ��r N).rd � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rC �gslib.utils.constantsr �gslib.utils.shim_utilr Zgslib.utils.text_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprf �_SET_DESCRIPTION�_GET_DESCRIPTION�join� _DESCRIPTIONrm rn ro �get_format_flag_caret�_GCLOUD_FORMAT_STRINGrq rr r r4 r4 r4 r5 �<module> sT � �� ��� __pycache__/bucketpolicyonly.cpython-39.pyc 0000644 00000015375 15025166704 0015025 0 ustar 00 a +(Wg�" � @ s d Z ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z ddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de! e Z"ee e"�Z#ee e �Z$ee e!�Z%ej&j'Z'e'j(Z(G dd� de �Z)dS )z+This module provides the command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)�ubla)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)� InsistOnOrOffz> gsutil bucketpolicyonly set (on|off) gs://<bucket_name>... z5 gsutil bucketpolicyonly get gs://<bucket_name>... � a� <B>SET</B> The ``bucketpolicyonly set`` command enables or disables the uniform bucket-level access feature on Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil bucketpolicyonly set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil bucketpolicyonly set off gs://redbucket gs://bluebucket a& <B>GET</B> The ``bucketpolicyonly get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket. <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil bucketpolicyonly get gs://redbucket gs://bluebucket a The Bucket Policy Only feature is now known as `uniform bucket-level access <https://cloud.google.com/storage/docs/uniform-bucket-level-access>`_. The ``bucketpolicyonly`` command is still supported, but we recommend using the equivalent ``ubla`` `command <https://cloud.google.com/storage/docs/gsutil/commands/ubla>`_. The ``bucketpolicyonly`` command is used to retrieve or configure the uniform bucket-level access setting of Cloud Storage buckets. This command has two sub-commands, ``get`` and ``set``. c @ s� e Zd ZdZejdededdddej gej e � d�ge ddd gd �e � � gd �d � Z ejdg d deeed �d�ZejjZejd jd Ze�dd�ejd jd< dd� Zdd� Zdd� Zdd� Z d d � Z d S ) �BucketPolicyOnlyCommandz/Implements the gsutil bucketpolicyonly command.Zbucketpolicyonly� � F� �mode�on�off)�choices��get�set) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz%Configure uniform bucket-level access)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr � zUniform bucket-level accesszBucket Policy Onlyc C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r3 �./opt/gsutil/gslib/commands/bucketpolicyonly.py�,_ValidateBucketListingRefAndReturnBucketName� s ��zDBucketPolicyOnlyCommand._ValidateBucketListingRefAndReturnBucketNamec C s� | � |� |j}| jj|jdg|jd�}|j}|j}t|�� d�|j d�}d}|j rb|j |d< d}|r~t d| j f i |��� d S ) z1Gets the Bucket Policy Only setting for a bucket.�iamConfiguration��fields�provider�/)�bucket�enabledr Z locked_timez LockedTime: {locked_time} z>Bucket Policy Only setting for {bucket}: Enabled: {enabled} N)r5 r. � gsutil_api� GetBucket� bucket_namer/ r6 �bucketPolicyOnly�str�rstripr< � lockedTime�print�format)r1 r2 � bucket_url�bucket_metadata� iam_configZbucket_policy_onlyr8 Zlocked_time_liner3 r3 r4 �_GetBucketPolicyOnly� s* � � ��z,BucketPolicyOnlyCommand._GetBucketPolicyOnlyc C s~ | � |� |j}t� }t� |_|dk|j_tj|d�}|dkrBdnd}td|t |�� d�f � | j j |j |dg|jd� d S ) z;Sets the Bucket Policy Only setting for a bucket on or off.r )r6 ZEnablingZ Disablingz %s Bucket Policy Only for %s...r: r6 r7 r )r5 r. �IamConfigurationValue�BucketPolicyOnlyValuer@ r< �apitools_messages�BucketrD rA rB r= � PatchBucketr? r/ )r1 r2 � setting_argrF rH rG Z setting_verbr3 r3 r4 �_SetBucketPolicyOnly� s � �z,BucketPolicyOnlyCommand._SetBucketPolicyOnlyc C s� | j �d�}|dvr td��d}g }d}|dkr8| j}n.|dkrf| j}| j �d�}t|d� |�|� d}| j }|s|| �� |D ],}| �|�}|D ]} d }|| g|�R � q�q�|s�tt t |� ��dS ) z;Handles bucketpolicyonly command on a Cloud Storage bucket.r r z&bucketpolicyonly only supports get|setNr r z-Only on and off values allowed for set optionFT) �args�popr rI rP r �append�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr �list) r1 � subcommand�subcommand_func�subcommand_argsrO � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr3 r3 r4 �_BucketPolicyOnly� s6 � z)BucketPolicyOnlyCommand._BucketPolicyOnlyc C s� | j jdd�tjkr.td�t�d| j ����| j d }| j dd� |dksT|d krzt j | j d � t j |gd � | �� n td | ��d S )z5Command entry point for the bucketpolicyonly command.r- )r9 r zBThe "%s" command can only be used with the Cloud Storage JSON API.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)r= �GetApiSelectorr �JSONr �join�textwrap�wrapr0 rQ � ParseSubOptsr �LogCommandParamsra r_ )r1 �action_subcommandr3 r3 r4 � RunCommand� s" ��� �z"BucketPolicyOnlyCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rd r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr Z UblaCommand�gcloud_storage_map�gcloud_commandZ format_flag�replacer5 rI rP r_ rk r3 r3 r3 r4 r \ sR ������� �$r )*ro � __future__r r �getoptrf �gslibr � gslib.commandr �gslib.command_argumentr �gslib.commandsr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rL �gslib.utils.constantsr Zgslib.utils.text_utilr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprq �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrv rx rw rM rJ rK r r3 r3 r3 r4 �<module> s> � � __pycache__/retention.cpython-39.pyc 0000644 00000050031 15025166704 0013421 0 ustar 00 a +(Wg w � @ s< d Z ddlmZ ddlZddlmZ ddlmZ ddlm Z ddlm Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl&m(Z( ddl&m)Z) ddl&m*Z* ddl&m+Z+ ddl&m,Z, dd l&m-Z- dd l&m.Z. dd l/m0Z0 dd l1m2Z2 d Z3d!Z4d"Z5d#Z6d$Z7d%Z8d&Z9d'Z:d(Z;d)Z<d*Z=d+Z>d,Z?d-Z@e3e4 e5 e6 e7 e8 e9 ZAe:e; e< e= e> e? e@ ZBeeAeB�ZCee3e:�ZDee4e;�ZEee5e<�ZFee6e=�ZGee7e>�ZHee8e?�ZIee9e@�ZJG d.d/� d/e �ZKdS )0zEImplementation of Retention Policy configuration command for buckets.� )�absolute_importN)�encoding)�metrics)�AccessDeniedException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�storage_v1_messages)�MetadataMessage)�GetCloudApiInstance)�NO_MAX)�PutToQueueWithTimeout)�ConfirmLockRequest)�ReleaseEventHoldFuncWrapper)�ReleaseTempHoldFuncWrapper)�RetentionInSeconds)�RetentionPolicyToString)�SetEventHoldFuncWrapper)�SetTempHoldFuncWrapper)�$UpdateObjectMetadataExceptionHandler)�GcloudStorageMap)�PreconditionsFromHeaderszA gsutil retention set <retention_period> gs://<bucket_name>... z0 gsutil retention clear gs://<bucket_name>... z. gsutil retention get gs://<bucket_name>... z/ gsutil retention lock gs://<bucket_name>... zF gsutil retention event-default (set|release) gs://<bucket_name>... zL gsutil retention event (set|release) gs://<bucket_name>/<object_name>... zK gsutil retention temp (set|release) gs://<bucket_name>/<object_name>... a� <B>SET</B> You can configure a data retention policy for a Cloud Storage bucket that governs how long objects in the bucket must be retained. You can also lock the data retention policy, permanently preventing the policy from being reduced or removed. For more information, see `Retention policies and Bucket Lock <https://cloud.google.com/storage/docs/bucket-lock>`_. The ``gsutil retention set`` command allows you to set or update the retention policy on one or more buckets. To remove an unlocked retention policy from one or more buckets, use the ``gsutil retention clear`` command. The ``set`` sub-command can set a retention policy with the following formats: <B>SET FORMATS</B> Formats for the ``set`` subcommand include: <number>s Specifies retention period of <number> seconds for objects in this bucket. <number>d Specifies retention period of <number> days for objects in this bucket. <number>m Specifies retention period of <number> months for objects in this bucket. <number>y Specifies retention period of <number> years for objects in this bucket. GCS JSON API accepts retention periods as number of seconds. Durations provided in terms of days, months or years are converted to their rough equivalent values in seconds, using the following conversions: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Retention periods must be greater than 0 and less than 100 years. Retention durations must be in only one form (seconds, days, months, or years), and not a combination of them. Note that while it is possible to specify retention durations shorter than a day (using seconds), enforcement of such retention periods is not guaranteed. Such durations may only be used for testing purposes. <B>EXAMPLES</B> Setting a retention policy with a duration of 1 year on a bucket: gsutil retention set 1y gs://my-bucket Setting a retention policy with a duration of 36 months on a bucket: gsutil retention set 36m gs://some-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the retention policy on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \ retention set 1y gs://my-bucket a4 <B>CLEAR</B> The ``gsutil retention clear`` command removes an unlocked retention policy from one or more buckets. You cannot remove or reduce the duration of a locked retention policy. <B>EXAMPLES</B> Clearing an unlocked retention policy from a bucket: gsutil retention clear gs://my-bucket z� <B>GET</B> The ``gsutil retention get`` command retrieves the retention policy for a given bucket and displays a human-readable representation of the configuration. a1 <B>LOCK</B> The ``gsutil retention lock`` command PERMANENTLY locks an unlocked retention policy on one or more buckets. CAUTION: A locked retention policy cannot be removed from a bucket or reduced in duration. Once locked, deleting the bucket is the only way to "remove" a retention policy. a� <B>EVENT-DEFAULT</B> The ``gsutil retention event-default`` command sets the default value for an event-based hold on one or more buckets. By setting the default event-based hold on a bucket, newly-created objects inherit that value as their event-based hold (it is not applied retroactively). <B>EXAMPLES</B> Setting the default event-based hold on a bucket: gsutil retention event-default set gs://my-bucket Releasing the default event-based hold on a bucket: gsutil retention event-default release gs://my-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the default event-based hold on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \ retention event-default set gs://my-bucket a[ <B>EVENT</B> The ``gsutil retention event`` command enables or disables an event-based hold on an object. <B>EXAMPLES</B> Setting the event-based hold on an object: gsutil retention event set gs://my-bucket/my-object Releasing the event-based hold on an object: gsutil retention event release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the event-based hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \ retention event set gs://my-bucket/my-object If you want to set or release an event-based hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets an event-based hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention event set gs://my-bucket/*.jpg aA <B>TEMP</B> The ``gsutil retention temp`` command enables or disables a temporary hold on an object. <B>EXAMPLES</B> Setting the temporary hold on an object: gsutil retention temp set gs://my-bucket/my-object Releasing the temporary hold on an object: gsutil retention temp release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the temporary hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \ retention temp set gs://my-bucket/my-object If you want to set or release a temporary hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets a temporary hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention temp set gs://bucket/*.jpg c s> e Zd ZdZejdg ededddej gej e � � ge � � ge � d�ge � � ge � � ge � � gd�e � � ge � � gd�e � � ge � � gd�d�d� Zejdg d d eeeeeeeed �d �Z� fd d�Zdd� Zdd� Zd,dd�Z dd� Z dd� Z dd� Z d d � Z d d � Z!d d!� Z"d"d#� Z#d$d%� Z$d&d'� Z%d(d)� Z&d*d+� Z'� Z(S )-�RetentionCommandz+Implementation of gsutil retention command.� retention� F� ��set�release)r$ �clear�get�lock� event-default�event�temp) �command_name_aliases�usage_synopsis�min_args�max_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz=Provides utilities to interact with Retention Policy feature.)r' r$ r& r( r) r* r+ )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s | j d dkr^tdtdddd�t| j d ��g| j dd � i d �ii d �}| j d d� | _ n�ttg d �i d �ttg d �i d �tg d �i d �d �i d �ttg d�i d �tg d�i d �d �i d �tg d�i d �tg d�i d �ttg d�i d �tg d�i d �d �i d �d�i d �}t� �|�S )Nr r$ �storage�buckets�updatez--retention-period={}sr" r! )�gcloud_command�flag_map)r= r> r? z--clear-retention-period)r= �objectsr? z--event-based-hold)r= rB r? z--no-event-based-holdr# )r= r> r? z--default-event-based-hold)r= r> r? z --no-default-event-based-hold)r= r> �describez --format=yaml(retentionPolicy)z--raw)r= r> r? z--lock-retention-period)r= rB r? z--temporary-hold)r= rB r? z--no-temporary-hold)r& r* r) r' r( r+ )�argsr �formatr �super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� �'/opt/gsutil/gslib/commands/retention.pyrG I s� �� ��� � ����������������Z�]z(RetentionCommand.get_gcloud_storage_argsc C s� | j �d�tjkr"td�| j���t| j�| _ | j � d�}| j dd� |dkrV| j }nr|dkrf| j}nb|dkrv| j}nR|d kr�| j}nB|d kr�| j}n2|d kr�| j}n"|d kr�| j}ntd �|| j���tj|g| jd� |� S )z.Command entry point for the retention command.�gsz�The {} command can only be used with the GCS JSON API. If you have only supplied hmac credentials in your boto file, please instead supply a credential type that can be used with the JSON API.r T)� check_argsr$ r& r' r( r) r* r+ zHInvalid subcommand "{}" for the {} command. See "gsutil help retention".)� subcommands�sub_opts)� gsutil_api�GetApiSelectorr �JSONr rE � command_namer �headers� preconditionsrD �pop� ParseSubOpts� _SetRetention�_ClearRetention� _GetRetention�_LockRetention�_DefaultEventHold� _EventHold� _TempHoldr �LogCommandParamsrQ )rH �action_subcommand�funcrL rL rM � RunCommand� s> � ���zRetentionCommand.RunCommandc C s� t | jjd�}d}|D ]P}| j|dgd�}|D ]6} | j} d}| j�|| � | jj| j ||| j |d� q.q|s|t t t |� ��d S )N)�meta_gen_matchF�id�� bucket_fieldsT)rW �provider�fields)r rW re �GetBucketUrlIterFromArg� storage_url�logger�inforR � PatchBucket� bucket_name�schemer r �list) rH �url_args�bucket_metadata_updaterj �log_msg_templaterW � some_matched�url_str� bucket_iter�blr�urlrL rL rM �BucketUpdateFunc� s$ � � z!RetentionCommand.BucketUpdateFuncNc C s� t | |d�}|j}| j�||� t�tj|j�}t | j j | j j d�}|j du rV|j |_ |j du rh|j|_ |j|j|j||j ||jdgd� t|jtt�� d�� dS )a� Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. )� thread_state)� gen_matchre Nrf )� generationrW ri rj )� message_time)r �expanded_storage_urlrm rn r � JsonToMessage�apitools_messages�Object�expanded_resultr rW r} re r~ �metageneration�PatchObjectMetadatarp � object_namerq r � status_queuer �time) rH Zpatch_obj_metadata� log_template�name_expansion_resultr| rR � exp_src_urlZcloud_obj_metadatarW rL rL rM �ObjectUpdateMetadataFunc� s2 �� � �z)RetentionCommand.ObjectUpdateMetadataFuncc C s, t | j| j| j| j|| j| j| jddgd� S )Nr~ r� )� all_versions�continue_on_error�bucket_listing_fields)r rU �debugrm rR �recursion_requestedr� �parallel_operations�rH rs rL rL rM � _GetObjectNameExpansionIterator! s �z0RetentionCommand._GetObjectNameExpansionIteratorc C s$ t | j| j| �� || j| j| jd�S )N)r� � project_id)r rU r� �GetSeekAheadGsutilApir� r� r� r� rL rL rM �"_GetSeekAheadNameExpansionIterator- s �z3RetentionCommand._GetSeekAheadNameExpansionIteratorc C sT t | jd �}tjj|d�}d}tj|d�}| jdd� }| j||ddg|d � dS ) z6Set retention retention_period on one or more buckets.r ��retentionPeriodz!Setting Retention Policy on %s...��retentionPolicyr" Nrf r� �rj ru )r rD r� �Bucket�RetentionPolicyValuer{ )rH �seconds�retention_policyru rt rs rL rL rM rZ 6 s ���z RetentionCommand._SetRetentionc C s> t jjdd�}d}t j|d�}| j}| j||ddg|d� dS ) z8Clear retention retention_period on one or more buckets.Nr� z"Clearing Retention Policy on %s...r� rf r� r� r )r� r� r� rD r{ )rH r� ru rt rs rL rL rM r[ G s ���z RetentionCommand._ClearRetentionc C s. | j | jd dgd�\}}tt|j|�� dS )z)Get Retention Policy for a single bucket.r r� rg )�GetSingleBucketUrlFromArgrD �printr r� )rH � bucket_url�bucket_metadatarL rL rM r\ U s � z RetentionCommand._GetRetentionc C s� | j }d}|D ]�}| j|dgd�}|D ]�}|j}d}| jj|j|jg d�d�}|jr\|jjsnt d� |j���q&|jj du r�| j � d|� q&t|j|j�r�| j �d |� | jj|j|j|jd � q&| j � d � |�� q&q|s�t tt|� ��d S ) z-Lock Retention Policy on one or more buckets.Frf rg T)rf r� r� )ri rj z7Bucket "{}" does not have an Unlocked Retention Policy.z+Retention Policy on "%s" is already locked.z!Locking Retention Policy on %s...)ri z& Abort Locking Retention Policy on {}r )rD rk rl rR � GetBucketrp rq r� r� r rE �isLockedrm �errorr rn �LockRetentionPolicyr� r rr )rH rs rv rw rx ry rz r� rL rL rM r] \ sH ���� � ��z RetentionCommand._LockRetentionc C s� d}| j rJ| j d �� dkr"d}n(| j d �� dkr:d}ntd�| j���|rRdnd }d �|�}tj|d �}| j d d� }| j||d dg|d� dS )z?Sets default value for Event-Based Hold on one or more buckets.Nr r$ Tr% FzeInvalid subcommand "{}" for the "retention event-default" command. See "gsutil help retention event".ZSettingZ Releasingz${} default Event-Based Hold on %s...)�defaultEventBasedHoldr" rf r� r� )rD �lowerr rE rQ r� r� r{ )rH �hold�verbru rt rs rL rL rM r^ � s. �� ��z"RetentionCommand._DefaultEventHoldc C s> d}d}| � |�}| jdd� }|r(tnt}| �|||� dS )z7Sets or unsets Event-Based Hold on one or more objects.r* z Event-Basedr" Nr )�_ProcessHoldArgsrD r r �_SetHold�rH �sub_command_name�sub_command_full_namer� rs �obj_metadata_update_wrapperrL rL rM r_ � s ��zRetentionCommand._EventHoldc C s> d}d}| � |�}| jdd� }|r(tnt}| �|||� dS )z5Sets or unsets Temporary Hold on one or more objects.r+ Z Temporaryr" Nr )r� rD r r r� r� rL rL rM r` � s ��zRetentionCommand._TempHoldc C sP d}| j d �� dkr d}n0| j d �� dkr4d}ntd�| j d ||���|S )a Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. Nr r$ Tr% FzWInvalid subcommand "{}" for the "retention {}" command. See "gsutil help retention {}".)rD r� r rE )rH r� r� rL rL rM r� � s ��z!RetentionCommand._ProcessHoldArgsc C s� t |�dkr@| js@t|d �}|�� r.|�� s@td�|d ���| �|�}| �|�}d| _ z| j ||t d|d� W n8 t y� } z |j dkr�| �� � W Y d}~n d}~0 0 | j s�td�|���dS ) ap Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" r" r z URL ({}) must name an objectT)� fail_on_error�seek_ahead_iteratori� Nz*{} Hold for some objects could not be set.)�lenr� r � IsCloudUrl�IsObjectr rE r� r� �everything_set_okay�Applyr r �status�_WarnServiceAccounts)rH r� rs r� rz �name_expansion_iteratorr� �erL rL rM r� � s4 � � ��zRetentionCommand._SetHold)N))�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rT r �%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text�_clear_help_text�_lock_help_text�_event_default_help_text�_event_help_text�_temp_help_text� help_specrG rd r{ r� r� r� rZ r[ r\ r] r^ r_ r` r� r� � __classcell__rL rL rJ rM r st ����� �� p* � ( % r )Lr� � __future__r r� �apitools.base.pyr �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.name_expansionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager � gslib.utils.cloud_api_helperr �gslib.utils.constantsr �&gslib.utils.parallelism_framework_utilr �gslib.utils.retention_utilr r r r r r r r �gslib.utils.shim_utilr � gslib.utils.translation_helperr � _SET_SYNOPSISZ_CLEAR_SYNOPSIS� _GET_SYNOPSISZ_LOCK_SYNOPSISZ_EVENT_DEFAULT_SYNOPSISZ_EVENT_SYNOPSISZ_TEMP_SYNOPSIS�_SET_DESCRIPTIONZ_CLEAR_DESCRIPTION�_GET_DESCRIPTIONZ_LOCK_DESCRIPTIONZ_EVENT_DEFAULT_DESCRIPTIONZ_EVENT_DESCRIPTIONZ_TEMP_DESCRIPTIONr� � _DESCRIPTIONr� r� r� r� r� r� r� r� r rL rL rL rM �<module> s� @ ��� ���� � __pycache__/mb.cpython-39.pyc 0000644 00000030360 15025166704 0012013 0 ustar 00 a +(WgkB � @ sx d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl'm(Z( d�)e�Z*de* dj)ed � Z+e�,d �Z-e�,d �Z.d Z/ej0j1Z1e1j2Z2G d d!� d!e �Z3dS )"z@Implementation of mb command for creating cloud storage buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�AccessDeniedException�BadRequestException)�Command)�CommandArgument)�VALID_RPO_VALUES)�VALID_RPO_VALUES_STRING)� ApiSelector)�CommandException)�InvalidUrlError)�StorageUrlFromString)�storage_v1_messages)�NO_MAX)�RetentionInSeconds)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� InsistOnOrOff)�NormalizeStorageClass)� ValidateCMEKz� gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>] [--autoclass] [--retention <time>] [--pap <setting>] [--placement <region1>,<region2>] [--rpo {}] gs://<bucket_name>... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Create one or more new buckets. Google Cloud Storage has a single namespace, so you are not allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "gsutil help naming"). If you don't specify a project ID or project number using the -p option, the buckets are created using the default project ID specified in your `gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. The -l option specifies the location for the buckets. Once a bucket is created in a given location, it cannot be moved to a different location. Instead, you need to create a new bucket, move the data over, and then delete the original bucket. <B>BUCKET STORAGE CLASSES</B> You can specify one of the `storage classes <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket with the -c option. Example: gsutil mb -c nearline gs://some-bucket See online documentation for `pricing <https://cloud.google.com/storage/pricing>`_ and `SLA <https://cloud.google.com/storage/sla>`_ details. If you don't specify a -c option, the bucket is created with the default storage class Standard Storage. <B>BUCKET LOCATIONS</B> You can specify one of the `available locations <https://cloud.google.com/storage/docs/locations>`_ for a bucket with the -l option. Examples: gsutil mb -l asia gs://some-bucket gsutil mb -c standard -l us-east1 gs://some-bucket If you don't specify a -l option, the bucket is created in the default location (US). <B>Retention Policy</B> You can specify retention period in one of the following formats: --retention <number>s Specifies retention period of <number> seconds for objects in this bucket. --retention <number>d Specifies retention period of <number> days for objects in this bucket. --retention <number>m Specifies retention period of <number> months for objects in this bucket. --retention <number>y Specifies retention period of <number> years for objects in this bucket. Examples: gsutil mb --retention 1y gs://some-bucket gsutil mb --retention 36m gs://some-bucket If you don't specify a --retention option, the bucket is created with no retention policy. <B>OPTIONS</B> --autoclass Enables the Autoclass feature that automatically sets object storage classes. -b <on|off> Specifies the uniform bucket-level access setting. When "on", ACLs assigned to objects in the bucket are not evaluated. Consequently, only IAM policies grant access to objects in these buckets. Default is "off". -c class Specifies the default storage class. Default is ``Standard``. See `Available storage classes <https://cloud.google.com/storage/docs/storage-classes#classes>`_ for a list of possible values. -k <key> Set the default KMS key using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -l location Can be any supported location. See https://cloud.google.com/storage/docs/locations for a discussion of this distinction. Default is US. Locations are case insensitive. -p project Specifies the project ID or project number to create the bucket under. -s class Same as -c. --retention time Specifies the retention policy. Default is no retention policy. This can only be set on gs:// buckets and requires using the JSON API. For more details about retention policy see "gsutil help retention" --pap setting Specifies the public access prevention setting. Valid values are "enforced" or "inherited". When "enforced", objects in this bucket cannot be made publicly accessible. Default is "inherited". --placement reg1,reg2 Two regions that form the custom dual-region. Only regions within the same continent are or will ever be valid. Invalid location pairs (such as mixed-continent, or with unsupported regions) will return an error. --rpo setting Specifies the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_. This flag is not valid for single-region buckets, and multi-region buckets only accept a value of DEFAULT. Valid values for dual region buckets are {rpo_values}. If unspecified, DEFAULT is applied for dual-region and multi-region buckets. )Z rpo_valuesz.^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$z[-_a-z0-9]{64}z --retentionc s� e Zd ZdZejdg d�ededg d�dddej ej gej e � � gd � Z ejdg d �d d ei d �Zeg d�deddd��ded�ded�ded�ded�ded dd ��d ed �eed �d ed!�i d"�Z� fd#d$�Zd%d&� Z� ZS )'� MbCommandz$Implementation of gsutil mb command.�mb)� makebucket� createbucket�md�mkdir� z b:c:l:p:s:k:)� autoclassz retention=zpap=z placement=zrpo=Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r r r r �locationZdraZdrasZreduced_availabilityZ durable_reduced_availability�rr�reduced_redundancy�standardz storage class�nearline�nl� command_helpz Make buckets)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�create�-bz --uniform-bucket-level-accessN)�on�off�-cz--default-storage-class�-kz--default-encryption-key�-lz --location�-pz --project�--papz--public-access-prevention)ZenforcedZ inherited� --placementz--retention-period�--rpoz--recovery-point-objective)�gcloud_command�flag_mapc sp d}|t | j�k r0| j| d tkr&q0|d7 }q|t | j�k rbttt| j| d ��d f| j|< t� �tj�S )Nr r �s) �len�sub_opts�_RETENTION_FLAG�strr �super�get_gcloud_storage_argsr �gcloud_storage_map)�selfZretention_arg_idx�� __class__� � /opt/gsutil/gslib/commands/mb.pyrO s � z!MbCommand.get_gcloud_storage_argsc C s� d}d}d}d}d}d}d}d}d} g } | j �r~| j D �]D\} } | dkrXd}| �| � q6| dkrx| }t|� | �| � q6| dkr�| }q6| dkr�t| d� | | _q6| d ks�| d kr�t| �}q6| tkr�t| �}q6| d k�r| �� } | t vr�t d � t | ���| �| � q6| d k�r*t | d� | dk}| �| � q6| dk�rD| }| �| � q6| dkr6| �d�}t|�dk�rrt d� | ���| �| � q6tj|| |d�} |�r�tjj|d�| _|�s�|�r�t� | _| j}|�r�t� |_||j_|�r�||_|�r�tj�� }||_|| _ |�rtj� � }||_ || _ | j D �]�}t!|�}|du�r^|j"dk�rJt d��tjj#|d�}|| _$| �r�| j%�&|j"�t'j(k�r�t d� d�)| ����|�*� �s�t d | ��t+�,|j-��r�t.�/|j-��r�t0d |j- ��| j1�2d |� z | j%j3|j-| j| |j"d � W n� t4�yf } zR|j5}d |v �rP|d!7 }|d"| 7 }| j�rF|d#| j 7 }t |��n� W Y d}~n�d}~0 t6�y� } zl|j7d$k�r�|j5d%k�r�|j"dk�r�|j-}||�8d&�d' d� }t d(�)t9�:d)||f ����n� W Y d}~n d}~0 0 �q d*S )+z'Command entry point for the mb command.FNz --autoclassTrA rB rC z/Invalid non-ASCII character found in project IDr@ z-srF z9Invalid value for --rpo. Must be one of: {}, provided: {}r= z,Only on and off values allowed for -b optionr> rD rE �,� zJPlease specify two regions separated by comma without space. Specified: {})r- �rpo� storageClass)�enabled�gsz7Retention policy can only be specified for GCS buckets.)�retentionPeriodzCThe {} option(s) can only be used for GCS Buckets with the JSON APIz, zIThe mb command requires a URL that specifies a bucket. "%s" is not valid.z Invalid bucket name in URL "%s"zCreating %s...)� project_id�metadata�provider�keyz* To authorize, run: gsutil kms authorizez \ -k %sz \ -p %si� ZDotfulBucketNameNotUnderTld�.r � z�Buckets with "." in the name must be valid DNS names. The bucket you are attempting to create (%s) is not a valid DNS name, because the final component (%s) is not currently a valid part of the top-level DNS tree.r );rK �appendr r r] r rL r �stripr r �formatr r �splitrJ �apitools_messages�Bucket�AutoclassValuer �IamConfigurationValue�iamConfiguration�BucketPolicyOnlyValue�bucketPolicyOnlyrZ �publicAccessPrevention�EncryptionValue�defaultKmsKeyName� encryption�CustomPlacementConfigValue� dataLocations�customPlacementConfig�argsr �scheme�RetentionPolicyValue�retentionPolicy� gsutil_api�GetApiSelectorr �JSON�join�IsBucket�BUCKET_NAME_RE�match� bucket_name�TOO_LONG_DNS_NAME_COMP�searchr �logger�info� CreateBucketr �reasonr �status�rfind�textwrap�wrap)rQ r �bucket_policy_only�kms_keyr- � storage_class�secondsZ placements�public_access_preventionrX Zjson_only_flags_in_command�o�a�bucket_metadata� iam_configrq Zplacement_configZbucket_url_str� bucket_url�retention_policy�e�messager� Z final_comprT rT rU � RunCommand s �� �� �� � ��� � �� � ���� zMbCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XMLr{ r �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r rL rP rO r� � __classcell__rT rT rR rU r � sn ������� r )4r� � __future__r r r r �rer� �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr Zgslib.commands.rpor r �gslib.cs_api_mapr �gslib.exceptionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr rg �gslib.utils.constantsr Zgslib.utils.retention_utilr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr r r � gslib.utils.encryption_helperr re r� r� �compiler~ r� rL rh rj rl r rT rT rT rU �<module> sR ��|�� __pycache__/requesterpays.cpython-39.pyc 0000644 00000013036 15025166704 0014332 0 ustar 00 a +(Wg ! � @ s( d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de e! Z"ee e"�Z#ee e!�Z$ee e �Z%de�&� d d Z'G dd� de �Z(d S ) zBImplementation of requesterpays configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz; gsutil requesterpays set (on|off) gs://<bucket_name>... z2 gsutil requesterpays get gs://<bucket_name>... � z� <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable Requester Pays for the specified bucket. z� <B>GET</B> The "get" sub-command gets the Requester Pays configuration for a bucket and displays whether or not it is enabled. a% The `Requester Pays <https://cloud.google.com/storage/docs/requester-pays>`_ feature enables you to configure a Google Cloud Storage bucket so that the requester pays all costs related to accessing the bucket and its objects. The gsutil requesterpays command has two sub-commands: z)--format=value[separator=": "](name.sub("z ", "gs://"),z,requester_pays.yesno("Enabled", "Disabled"))c @ s� e Zd ZdZejdededdddej gej e dddgd �e � � ge � � gd �d � Z ej dg d d eeed�d�Zeedddegi dd�eeg d�i d�eg d�i d�d�i d�d�i d�Zdd� Zdd � Zd d � Zd d � Zd!S )"�RequesterPaysCommandz/Implementation of gsutil requesterpays command.Z requesterpays� � F�mode�on�off)�choices)�set�get) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz8Enable or disable requester pays for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listT)�gcloud_command�flag_map�supports_output_translation)r, r- �updatez--requester-pays)r/ r0 )r, r- r2 z--no-requester-pays�r r c C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r9 �+/opt/gsutil/gslib/commands/requesterpays.py�_CalculateUrlsStartArg� s z+RequesterPaysCommand._CalculateUrlsStartArgc C s� | j d �� }|dvr$td| j ��| j dd� }|s>| �� d}|D ]�}| j|dgd�}|D ]p}|j}d }tjtj� � d �}|d kr�| j � d |� d |j _ n| j � d |� d|j _ | jj|j||jdgd� q^qF|s�ttt|� ��dS )z.Gets requesterpays configuration for a bucket.r r3 z,Argument to "%s set" must be either <on|off>r4 NF�id�� bucket_fieldsT)�billingr z!Enabling requester pays for %s...z"Disabling requester pays for %s...)�provider�fields)r5 r7 r � command_namer6 �GetBucketUrlIterFromArg� storage_url�apitools_messages�Bucket� BillingValue�logger�infor? � requesterPays� gsutil_api� PatchBucket� bucket_name�schemer r. ) r8 �requesterpays_arg�url_args� some_matched�url_str� bucket_iter�blr�url�bucket_metadatar9 r9 r: �_SetRequesterPays� s: �� � z&RequesterPaysCommand._SetRequesterPaysc C s� | j }d}|D ]^}| j|dgd�}|D ]D}d}|jjrV|jjjrVtd|j�d� � q&td|j�d� � q&q|s�tt t |� ��dS ) z9Gets requesterpays configuration for one or more buckets.Fr? r= Tz %s: Enabled�/z %s: DisabledN) r5 rC � root_objectr? rJ �print� url_string�rstripr r r. )r8 rP rQ rR rS rT r9 r9 r: �_GetRequesterPays� s �z&RequesterPaysCommand._GetRequesterPaysc C s� | j �d�}|dkr*| j}tj|gd� nL|dkr`| j}| j d �� }|dv rvtj||gd� ntd|| j| jf ��|� dS )z2Command entry point for the requesterpays command.r r )� subcommandsr r3 zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r5 �popr] r �LogCommandParamsrW r7 r rB )r8 �action_subcommand�funcrO r9 r9 r: � RunCommand� s$ � ��z RequesterPaysCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr; rW r] rc r9 r9 r9 r: r M sn ������ �������"�&!r N))rg � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rE �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripri �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrn ro rp �get_format_flag_caretrr r r9 r9 r9 r: �<module> sH �� ��� __pycache__/logging.cpython-39.pyc 0000644 00000022054 15025166704 0013044 0 ustar 00 a +(Wg�3 � @ sT d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" dZ#dZ$e#e$�%d� d Z&dZ'dZ(de' e( d Z)ee&e)�Z*ee$e(�Z+ee#e'�Z,G d d � d e �Z-dS ) z<Implementation of logging configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMap)� text_utilz� gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... gsutil logging set off gs://<bucket_name>... z) gsutil logging get gs://<bucket_name> � a� <B>SET</B> The ``set`` sub-command has two sub-commands: <B>ON</B> The ``gsutil logging set on`` command enables usage and storage logging for the buckets named by the specified URLs, outputting log files to the bucket specified with the ``-b`` flag. Cloud Storage doesn't validate the existence of the output bucket, so users should ensure it already exists, and all URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). The optional ``-o`` flag specifies the prefix for log object names. The default prefix is the bucket name. For example, the command: gsutil logging set on -b gs://my_logging_bucket -o UsageLog \ gs://my_bucket1 gs://my_bucket2 causes all read and write activity to objects in ``gs://mybucket1`` and ``gs://mybucket2`` to be logged to objects prefixed with the name ``UsageLog``, with those log objects written to the bucket ``gs://my_logging_bucket``. In addition to enabling logging on your bucket(s), you also need to grant cloud-storage-analytics@google.com write access to the log bucket, using this command: gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket Note that log data may contain sensitive information, so you should make sure to set an appropriate default bucket ACL to protect that data. (See "gsutil help defacl".) <B>OFF</B> This command disables usage and storage logging for the buckets named by the specified URLs. All URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). No logging data is removed from the log buckets when you disable logging, but Google Cloud Storage stops delivering new logs once you have run this command. a6 <B>GET</B> If logging is enabled for the specified bucket url, the server responds with a JSON document that looks something like this: { "logBucket": "my_logging_bucket", "logObjectPrefix": "UsageLog" } You can download log data from your log bucket using the gsutil cp command. aM Google Cloud Storage offers `usage logs and storage logs <https://cloud.google.com/storage/docs/access-logs>`_ in the form of CSV files that you can download and view. Usage logs provide information for all of the requests made on a specified bucket and are created hourly. Storage logs provide information about the storage consumption of that bucket for the last day and are created daily. Once set up, usage logs and storage logs are automatically created as new objects in a bucket that you specify. Usage logs and storage logs are subject to the same pricing as other objects stored in Cloud Storage. For a complete list of usage log fields and storage data fields, see `Usage and storage log format <https://cloud.google.com/storage/docs/access-logs#format>`_. The ``logging`` command has two sub-commands: a� <B>OPTIONS</B> -b bucket_name Specifies the bucket that stores the generated logs. This flag is only available for the ``set on`` command and is required for that command. -o log_prefix Specifies a common prefix for the names of generated logs. This flag is only available for the ``set on`` command and is optional for that command. c @ s� e Zd ZdZejdg d�ededdddej ej gej e dd d gd �e � � gd � Z ejdg d �ddeeed�d�Zeeg d�i d�eeg d�ed�ed�d�d�eg d�i d�d�i d�d�i d�Zdd� Zd d � Zd d � Zd d!� Zd"S )#�LoggingCommandz)Implementation of gsutil logging command.�logging)�disablelogging� enablelogging� getlogging� zb:o:Fr �mode�on�off)�choices) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)Z loggingconfigZlogs�logr r r � command_helpz(Configure or retrieve logging on buckets)�get�set)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describezk--format=gsutiljson[key=logging_config,empty=' has no logging configuration.',empty_prefix_key=storage_url])�gcloud_command�flag_map)r5 r6 �updatez --log-bucketz--log-object-prefix)�-b�-o)r5 r6 r: z--clear-log-bucketz--clear-log-object-prefix)r r c C s� | j | jd dgd�\}}|jdkrBtj| jj||jd�dd� n<|jrp|jjrp|jj rpt�t t � |j��� nt�d| � dS ) z(Gets logging configuration for a bucket.r r �� bucket_fields�s3)�provider� )�endz %s has no logging configuration.) �GetSingleBucketUrlFromArg�args�schemer � print_to_fd� gsutil_api�XmlPassThroughGetLoggingr � logBucket�logObjectPrefix�strr � MessageToJson)�self� bucket_url�bucket_metadata� rP �%/opt/gsutil/gslib/commands/logging.py�_Get� s" � ����zLoggingCommand._Getc C s� t | j�std��d}d}| jD ]$\}}|dkr8t|�}|dkr |}q |sRtd��|�� sbtd��d}| jD ]t}| j|dgd �}|D ]Z}|j} d }| j� d |� t j j |j |p�| j d �} t j | d �} | jj| j | | jdgd� q�ql|s�ttt| j� ��dS )z+Enables logging configuration for a bucket.z8"logging set on" command spanning providers not allowed.Nr; r< z2"logging set on" requires '-b <log_bucket>' optionz$-b option must specify a bucket URL.F�idr= TzEnabling logging on %s...)rI rJ �r �r@ �fieldsr )r rD r �sub_optsr �IsBucket�GetBucketUrlIterFromArg� storage_url�logger�info�apitools_messages�Bucket� LoggingValue� bucket_namerG � PatchBucketrE r �list) rM Ztarget_bucket_url� target_prefix�opt�opt_arg� some_matched�url_str� bucket_iter�blr�urlr rO rP rP rQ �_Enable� sB � � zLoggingCommand._Enablec C s� d}| j D ]f}| j|dgd�}|D ]L}|j}d}| j�d|� tj�� }tj|d�}| jj |j ||j dgd� q"q |s�t t t| j � ��dS ) z,Disables logging configuration for a bucket.FrS r= TzDisabling logging on %s...rT rU r )rD rY rZ r[ r\ r] r^ r_ rG ra r` rE r r rb )rM rf rg rh ri rj r rO rP rP rQ �_Disable# s" � zLoggingCommand._Disablec C s� | j �d�}|dkr*| j}tj|gd� n�|dkr�| j �d�}| j sL| �� |dkrl| j}tj||gd� q�|dkr�| j}tj||gd� q�td|| j |f ��ntd|| j f ��| j d d � tj| j d � |� dS ) z,Command entry point for the logging command.r r- )� subcommandsr. r r zKInvalid subcommand "%s" for the "%s %s" command. See "gsutil help logging".zFInvalid subcommand "%s" for the %s command. See "gsutil help logging".T)� check_args)rW ) rD �poprR r �LogCommandParams�$RaiseWrongNumberOfArgumentsExceptionrk rl r � command_name� ParseSubOptsrW )rM �action_subcommand�funcZstate_subcommandrP rP rQ � RunCommand8 s< �� ��� zLoggingCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr r �gcloud_storage_maprR rk rl rv rP rP rP rQ r � sj ����� ������+�/*r ).rz � __future__r r r r �sys�apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr r] �gslib.utils.constantsr �gslib.utils.shim_utilr r Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripr| �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� r rP rP rP rQ �<module> sH *��� __pycache__/version.cpython-39.pyc 0000644 00000012404 15025166704 0013101 0 ustar 00 a +(Wgy � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ dZde d Z G dd� de�Z dd� Z dS )�)Implementation of gsutil version command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)� system_util)�GetFriendlyConfigFilePaths)�UsingCrcmodExtension)�UTF8)�GetMd5)�$CheckMultiprocessingAvailableAndInitz gsutil version z <B>SYNOPSIS</B> aS <B>DESCRIPTION</B> Prints information about the version of gsutil. <B>OPTIONS</B> -l Prints additional information, such as the version of Python being used, the version of the Boto library, a checksum of the code, the path to gsutil, and the path to gsutil's configuration file. c @ sV e Zd ZdZejddgeddddddd� Zejddgdd e i d �Z d d � Z d d� Z dS )�VersionCommandr �version�verr �lF)�command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg� command_helpz Print version info about gsutil)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s d}| j r&| j D ]\}}|dkrd}qd�t� �}tj}z | �� }W n tyZ d}Y n0 ||krjd}nd| }tj� dtj � |r�d }tj� |j ||t j tj�d d �d t�� t�� f t� jt�� t�� |t� p�tjt� tjtj t j � d dd�d�� dS )z,Command entry point for the version command.Fz-lTz, z MISSING FILES�OKz!= %szgsutil version: %s a� checksum: {checksum} ({checksum_ok}) boto version: {boto_version} python version: {python_version} OS: {os_version} multiprocessing available: {multiprocessing_available} using cloud sdk: {cloud_sdk} pass cloud sdk credentials to gsutil: {cloud_sdk_credentials} config path(s): {config_paths} gsutil path: {gsutil_path} compiled crcmod: {compiled_crcmod} installed via package manager: {is_package_install} editable install: {is_editable_install} shim enabled: {is_shim_enabled} � � z%s %s�GSUtil�use_gcloud_storage)�checksumZ checksum_ok� boto_version�python_version� os_versionZmultiprocessing_availableZ cloud_sdkZcloud_sdk_credentials� config_paths� gsutil_pathZcompiled_crcmodZis_package_installZis_editable_installZis_shim_enabledr ) �sub_opts�joinr �gslib�CHECKSUM�_ComputeCodeChecksum�IOError�sys�stdout�write�VERSION�format�boto� __version__r �replace�platform�system�releaser � is_availabler �InvokedViaCloudSdk�CloudSdkCredPassingEnabled�"GetCloudSdkGsutilWrapperScriptPath� GSUTIL_PATHr �IS_PACKAGE_INSTALL�IS_EDITABLE_INSTALL�config�getbool) �selfZ long_form�o�_r* Zshipped_checksumZ cur_checksumZchecksum_ok_strZlong_form_output� rI �%/opt/gsutil/gslib/commands/version.py� RunCommandT sP � ���zVersionCommand.RunCommandc C s� t jr dS t� }t jg}t�t j�D ]2\}}}|D ]"}|�d�r2|�tj � ||�� q2q$t |�D ]z}t j r�t|d�}|�� }t�dd|�}|�|� |�� q`t|dtd�}|�� }t�dd|�}|�|�t�� |�� q`|�� S ) af Computes a checksum of gsutil code. This checksum can be used to determine if users locally modified gsutil when requesting support. (It's fine for users to make local mods, but when users ask for support we ask them to run a stock version of gsutil so we can reduce possible variables.) Returns: MD5 checksum of gsutil code. Z.PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMSz.py�rbz (\r\n|\r|\n)� �r)�encodingr" )r. rB r rA �os�walk� GSLIB_DIR�endswith�append�pathr- �sorted�six�PY2�open�read�re�sub�update�closer �encode� hexdigest) rF �mZfiles_to_checksum�rootrH �files�filepath�f�contentrI rI rJ r0 � s* z#VersionCommand._ComputeCodeChecksumN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrK r0 rI rI rI rJ r ; s, � � =r c C sZ t j} t�� rVtj�ddd�}| �|�rVtj�| d| �|�� dd�}tj� |�rV|S dS )a If gsutil was invoked via the Cloud SDK, find its gsutil wrapper script. Returns: (str) The path to the Cloud SDK's gsutil wrapper script, or an empty string if gsutil was not invoked via the Cloud SDK or the wrapper script could not be found at its expected path. r: �gsutilr �binr# ) r. rA r r> rP rU r- rS �rfind�exists)r+ Zplatform_path_suffixZbin_pathrI rI rJ r@ � s � r@ ) rj � __future__r r r r rP r: r[ r2 rW r7 �crcmodr. � gslib.commandr Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr �gslib.utils.hashing_helperr �&gslib.utils.parallelism_framework_utilr rl ro r r@ rI rI rI rJ �<module> s6 ��| __pycache__/acl.cpython-39.pyc 0000644 00000045414 15025166704 0012162 0 ustar 00 a +(Wg#_ � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l m Z dd l mZ dd l mZ dd l mZ ddl mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl m!Z! ddl"m#Z$ ddl%m&Z& ddl'm(Z( dd l)m*Z* dd l+m,Z, dd l+m-Z- d Z.d Z/d!Z0d"Z1d#Z2d$Z3e.e/�4d%� e0�4d%� d& Z5d'd%�6e1e2e3g� Z7e e5e7�Z8e e/e1�Z9e e.e2�Z:e e0e3�Z;d(d)� Z<d.d*d+�Z=G d,d-� d-e�Z>dS )/z:Implementation of acl command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)� gcs_json_api)�AccessDeniedException)�BadRequestException)�PreconditionException)� Preconditions)�ServiceException)�Command)�SetAclExceptionHandler)�SetAclFuncWrapper)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText��StorageUrlFromString)�UrlsAreForSingleProvider)�)RaiseErrorIfUrlsAreMixOfBucketsAndObjects)�storage_v1_messages)� acl_helper)�NO_MAX)�Retry)�GcloudStorageFlag)�GcloudStorageMapzG gsutil acl set [-f] [-r] [-a] (<file-path>|<predefined-acl>) url... z gsutil acl get url aK gsutil acl ch [-f] [-r] <grant>... url... where each <grant> is one of the following forms: -u <id>|<email>:<permission> -g <id>|<email>|<domain>|All|AllAuth:<permission> -p (viewers|editors|owners)-<project number>:<permission> -d <id>|<email>|<domain>|All|AllAuth|(viewers|editors|owners)-<project number> z� <B>GET</B> The ``acl get`` command gets the ACL text for a bucket or object, which you can save and edit for the acl set command. aU <B>SET</B> The ``acl set`` command allows you to set an Access Control List on one or more buckets and objects. As part of the command, you must specify either a predefined ACL or the path to a file that contains ACL text. The simplest way to use the ``acl set`` command is to specify one of the predefined ACLs, e.g.,: gsutil acl set private gs://example-bucket/example-object If you want to make an object or bucket publicly readable or writable, it is recommended to use ``acl ch``, to avoid accidentally removing OWNER permissions. See the ``acl ch`` section for details. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. If you want to define more fine-grained control over your data, you can retrieve an ACL using the ``acl get`` command, save the output to a file, edit the file, and then use the ``acl set`` command to set that ACL on the buckets and/or objects. For example: gsutil acl get gs://bucket/file.txt > acl.txt Make changes to acl.txt such as adding an additional grant, then: gsutil acl set acl.txt gs://cats/file.txt Note that you can set an ACL on multiple buckets or objects at once. For example, to set ACLs on all .jpg files found in a bucket: gsutil acl set acl.txt gs://bucket/**.jpg If you have a large number of ACLs to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m acl set acl.txt gs://bucket/**.jpg Note that multi-threading/multi-processing is only done when the named URLs refer to objects, which happens either if you name specific objects or if you enumerate objects by using an object wildcard or specifying the acl -r flag. <B>SET OPTIONS</B> The "set" sub-command has the following options -R, -r Performs "acl set" request recursively, to all objects under the specified URL. -a Performs "acl set" request on all object versions. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. If some of the ACLs couldn't be set, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m acl...". a <B>CH</B> The "acl ch" (or "acl change") command updates access control lists, similar in spirit to the Linux chmod command. You can specify multiple access grant additions and deletions in a single command run; all changes will be made atomically to each object in turn. For example, if the command requests deleting one grant and adding a different grant, the ACLs being updated will never be left in an intermediate state where one grant has been deleted but the second grant not yet added. Each change specifies a user or group grant to add or delete, and for grant additions, one of R, W, O (for the permission to be granted). A more formal description is provided in a later section; below we provide examples. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Grant anyone on the internet READ access to the object example-object: gsutil acl ch -u allUsers:R gs://example-bucket/example-object NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Grant the user john.doe@example.com READ access to all objects in example-bucket that begin with folder/: gsutil acl ch -r -u john.doe@example.com:R gs://example-bucket/folder/ Grant the group admins@example.com OWNER access to all jpg files in example-bucket: gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg Grant the owners of project example-project WRITE access to the bucket example-bucket: gsutil acl ch -p owners-example-project:W gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. Remove access to the bucket example-bucket for the viewers of project number 12345: gsutil acl ch -d viewers-12345 gs://example-bucket NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in the given project. Attempts to do so will appear to succeed, but the service will add the project owners group into the new set of ACLs before applying it. Note that removing a project requires you to reference the project by its number (which you can see with the acl get command) as opposed to its project ID string. Grant the service account foo@developer.gserviceaccount.com WRITE access to the bucket example-bucket: gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket Grant all users from the `G Suite <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ access to the bucket gcs.my-domain.org: gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org Remove any current access by john.doe@example.com from the bucket example-bucket: gsutil acl ch -d john.doe@example.com gs://example-bucket If you have a large number of objects to update, enabling multi-threading with the gsutil -m flag can significantly improve performance. The following command adds OWNER for admin@example.org using multi-threading: gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket Grant READ access to everyone from my-domain.org and to all authenticated users, and grant OWNER to admin@mydomain.org, for the buckets my-bucket and my-other-bucket, with multi-threading enabled: gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \ -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket <B>CH ROLES</B> You may specify the following roles with either their shorthand or their full name: R: READ W: WRITE O: OWNER For more information on these roles and the access they grant, see the permissions section of the `Access Control Lists page <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. <B>CH ENTITIES</B> There are four different entity types: Users, Groups, All Authenticated Users, and All Users. Users are added with -u and a plain ID or email address, as in "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. Groups are like users, but specified with the -g flag, as in "-g power-users@example.com:O". Groups may also be specified as a full domain, as in "-g my-company.com:r". allAuthenticatedUsers and allUsers are specified directly, as in "-g allUsers:R" or "-g allAuthenticatedUsers:O". These are case insensitive, and may be shortened to "all" and "allauth", respectively. Removing roles is specified with the -d flag and an ID, email address, domain, or one of allUsers or allAuthenticatedUsers. Many entities' roles can be specified on the same command line, allowing bundled changes to be executed in a single run. This will reduce the number of requests made to the server. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -R, -r Performs acl ch request recursively, to all objects under the specified URL. -u Add or modify a user entity's role. � z z+ The acl command has three sub-commands: c C s | j �d|� d| _d S )NzEncountered a problem: %sF)�logger�error�everything_set_okay)�cls� exception� r% �!/opt/gsutil/gslib/commands/acl.py�_ApplyExceptionHandler" s r' c C s | j ||d� d S )N)� thread_state)�ApplyAclChanges)r# Zurl_or_expansion_resultr( r% r% r& �_ApplyAclChangesWrapper' s r* c s� e Zd ZdZejdg d�ededdddej ej gej e � � e � � ge �d�ge � � gd�d � Zejdg d �d d eeeed �d�Zdd� Z� fdd�Zdd� Zdd� Zdd� Zdd� Ze e ddd �d&d d ��Z e e ddd �d d!� �Z d"d#� Z!d$d%� Z"� Z#S )'� AclCommandz%Implementation of gsutil acl command.�acl)�getacl�setacl�chacl� z afRrg:u:d:p:F� )�set�get�ch) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r- r. �chmodr/ � command_helpz-Get, set, or change bucket and/or object ACLs)r3 r2 r4 )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C sZ dd� | j D �}d}| jD ]\}}|dv rd} q4qt||� |d �� rR|sRdS dS d S ) Nc S s g | ] }t |��qS r% r )�.0�urlr% r% r& � <listcomp>R � z6AclCommand._get_shim_command_group.<locals>.<listcomp>F)�-r�-RTr �buckets�objects)�args�sub_optsr �IsBucket)�selfZobject_or_bucket_urls�recurseZflag_key�_r% r% r& �_get_shim_command_groupQ s z"AclCommand._get_shim_command_groupc sN | j �d�}|dkrFt| j d ��� r,d}nd}td|ddgi d�}n�|d kr�| �� | j �d�}tj�|�rxd | }n"|t j v r�t j | }n|}d | }| � � }td|d g|g t d �t d�t d�t d�d�d�}nl|dk�rB| �� t �| j�| _| � � }td|d gt d�t d�t d�t d�t d �t d�t d�t d�d�d�}t� �|�S )Nr r3 rO rN �storageZdescribez --format=multi(acl:format=json))�gcloud_command�flag_mapr2 z --acl-file=z--predefined-acl=�updatez--all-versionsz--continue-on-errorz --recursive)�-a�-frM rL r4 z--add-acl-grantz--remove-acl-grant)�-g�-p�-u�-dr[ r\ rM rL )rP �popr �IsObjectr � ParseSubOpts�os�path�isfiler �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONrV r r Ztranslate_sub_opts_for_shimrQ �super�get_gcloud_storage_args)rS � sub_commandZ command_group�gcloud_storage_mapZacl_file_or_predefined_aclZacl_flag�predefined_acl�� __class__r% r& ri _ sf �� ����� �� z"AclCommand.get_gcloud_storage_argsc C s6 | j s| �� | j d �� dks*| jdkr.dS dS d S )Nr r2 r. r1 )rP �$RaiseWrongNumberOfArgumentsException�lower�command_alias_used)rS r% r% r& �_CalculateUrlsStartArg� s z!AclCommand._CalculateUrlsStartArgc C s� d| _ | jr\| jD ]H\}}|dkr*d| _q|dkr:d| _ q|dksJ|dkrRd| _q| �� qz| �tt� W n. ty� } z| � � � W Y d}~n d}~0 0 | j s�t d��dS ) z>Parses options and sets ACLs on the specified buckets/objects.Fr[ Tr\ rL rM N�'ACLs for some objects could not be set.) �continue_on_errorrQ � all_versions�recursion_requested� RaiseInvalidArgumentException�SetAclCommandHelperr r r �_WarnServiceAccountsr" r )rS �o�unused_aZunused_er% r% r& �_SetAcl� s" zAclCommand._SetAclc C sb d| _ g | _d| _| jr�| jD ]�\}}|dkr6d| _q |dkrjd|v rNtd��| j�tj|tjj d�� q |dkr�| j�tj|tjj d�� q |d kr�| j�tj|tjj d�� q |d kr�| j�t� |�� q |d ks�|d kr�d| _ q | �� q | js�td ��t| j��r t| jd �jdk�r0td�| j���d| _| jtt| jg d�d� | j�s^td��dS )zAParses options and changes ACLs on the specified buckets/objects.TFr\ r] zgserviceaccount.comznService accounts are considered users, not groups; please use "gsutil acl ch -u" instead of "gsutil acl ch -g")� scope_typer^ r_ r` rL rM zFPlease specify at least one access change with the -g, -u, or -d flagsr �gsz2The "{0}" command can only be used with gs:// URLs�r, � generation�metageneration)� object_fieldsrs N)Zparse_versions�changesrt rQ r �appendr Z AclChangeZ ChangeTypeZGROUP�PROJECT�USERZAclDelrv rw r rP r �scheme�format� command_namer" � ApplyAclFuncr* r' )rS rz �ar% r% r& �_ChAcl� s\ ���� ����zAclCommand._ChAclc C s | � � td| ��d S )NzTFailed to set acl for %s. Please ensure you have OWNER-role access to this resource.)ry r )rS rI r% r% r& �_RaiseForAccessDenied� s �z AclCommand._RaiseForAccessDenied� )�tries� timeout_secsNc C s� |r |}n| j }|j}|�� r>|j|j|jddgd�}|j}n |�� r\t� t j |j �}|j}|sj| � |� | �||�dkr�| j�d|� dS z�|�� r�t|jd�}t j|d�} |j|j| ||jd gd � nvt|j|jd �}t j |d�} z&|j|j|j| ||j|jd gd � W n2 t�y> } z| �||� W Y d} ~ n d} ~ 0 0 | j�d |� W n� t�y� } z tdt | � ��W Y d} ~ ndd} ~ 0 t �y� | � |� Y n@ t�y� } z&|�� �r�tt | ���| �W Y d} ~ n d} ~ 0 0 dS )z�Applies the changes in self.changes to the provided URL. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, gsutil Cloud API instance to apply the changes. r, r� ��provider�fieldsr �No changes to %sN)�meta_gen_match�r, �id)� preconditionsr� r� �� gen_matchr� �r� r� r� r� zUpdated ACL on %sz$Received bad request from server: %s) � gsutil_api�expanded_storage_urlrR � GetBucket� bucket_namer� r, rb r � JsonToMessage�apitools_messages�Object�expanded_resultr� �$_ApplyAclChangesAndReturnChangeCountr �infor r� �Bucket� PatchBucketr� �PatchObjectMetadata� object_namer �(_RefetchObjectMetadataAndApplyAclChangesr r �strr ) rS �name_expansion_resultr( r� rI �bucket� current_acl� gcs_objectr� �bucket_metadata�object_metadata�er% r% r& r) � sl �� �� � "$ zAclCommand.ApplyAclChangesc C s� |j |j|j|jg d�d�}|j}| �||�dkrD| j�d|� dS tj |d�}t |j |j d�}|j |j|j|||j|j dgd � dS ) z<Reattempts object ACL changes after a PreconditionException.r r� r r� Nr� r� r� r� )�GetObjectMetadatar� r� r� r, r� r r� r� r� r r� r� r� )rS rI r� r� r� r� r� r% r% r& r� 3 s, � ��z3AclCommand._RefetchObjectMetadataAndApplyAclChangesc C s* d}| j D ]}||�||d| j�7 }q |S )Nr r, )r� �Executer )rS � storage_url� acl_messageZmodification_count�changer% r% r& r� L s �z/AclCommand._ApplyAclChangesAndReturnChangeCountc C s� | j �d�}| jdd� tj| jd� d| _|dkrTtj|gd� | �| j d � nR|dkrttj|gd� | �� n2|d v r�tj|gd� | � � nt d || j f ��dS ) z(Command entry point for the acl command.r T)� check_args)rQ Fr3 )� subcommandsr2 )r4 r� zBInvalid subcommand "%s" for the %s command. See "gsutil help acl".) rP ra rc r �LogCommandParamsrQ �def_acl�GetAndPrintAclr| r� r r� )rS Zaction_subcommandr% r% r& � RunCommandS s& ��zAclCommand.RunCommand)N)$�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr Z MakeFileURLOrCannedACLArgumentZ MakeZeroOrMoreCloudURLsArgumentZMakeNCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrV ri rr r| r� r� r r r) r r� r� r� � __classcell__r% r% rm r& r+ + sV � ���� <0 E r+ )N)?r� � __future__r r r r rd �apitools.base.pyr �gslibr r �gslib.cloud_apir r r r r � gslib.commandr r r Zgslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Z _SET_SYNOPSISZ _GET_SYNOPSISZ _CH_SYNOPSISZ_GET_DESCRIPTIONZ_SET_DESCRIPTIONZ_CH_DESCRIPTION�lstripr� �joinZ _DESCRIPTIONr� r� r� r� r' r* r+ r% r% r% r& �<module> sf = ��� ubla.py 0000644 00000024064 15025166704 0006055 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistOnOrOff from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil ubla set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil ubla get bucket_url... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``ubla set`` command enables or disables uniform bucket-level access for Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil ubla set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil ubla set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``ubla get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket(s). <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil ubla get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``ubla`` command is used to retrieve or configure the `uniform bucket-level access <https://cloud.google.com/storage/docs/bucket-policy-only>`_ setting of Cloud Storage bucket(s). This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue uniformBucketLevelAccessValue = IamConfigurationValue.BucketPolicyOnlyValue _GCLOUD_FORMAT_STRING = ( '--format=' + 'multi[terminator="' + shim_util.get_format_flag_newline() + '"](name:format="value(format(\'Uniform bucket-level' + ' access setting for gs://{}:\'))",' + ' iamConfiguration.uniformBucketLevelAccess.enabled.yesno(no="False")' + ':format="value[terminator=\'' + shim_util.get_format_flag_newline() + '\'](format(\' Enabled: {}\'))",' + ' iamConfiguration.uniformBucketLevelAccess.lockedTime.sub("T", " ")' + ':format="value(format(\' LockedTime: {}\'))")') class UblaCommand(Command): """Implements the gsutil ubla command.""" command_spec = Command.CreateCommandSpec( 'ubla', command_name_aliases=['uniformbucketlevelaccess'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ubla', help_name_aliases=['uniformbucketlevelaccess'], help_type='command_help', help_one_line_summary='Configure Uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--uniform-bucket-level-access', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-uniform-bucket-level-access', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetUbla(self, blr): """Gets the Uniform bucket-level access setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. uniform_bucket_level_access = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': uniform_bucket_level_access.enabled } locked_time_line = '' if uniform_bucket_level_access.lockedTime: fields['locked_time'] = uniform_bucket_level_access.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if uniform_bucket_level_access: print(('Uniform bucket-level access setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(**fields)) def _SetUbla(self, blr, setting_arg): """Sets the Uniform bucket-level access setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. iam_config.bucketPolicyOnly = uniformBucketLevelAccessValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Uniform bucket-level access for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Ubla(self): """Handles ubla command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('ubla only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetUbla elif subcommand == 'set': subcommand_func = self._SetUbla setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the ubla command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Ubla() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) autoclass.py 0000644 00000021613 15025166704 0007125 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the autoclass command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil autoclass set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil autoclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command requires an additional sub-command, either ``on`` or ``off``, which enables or disables Autoclass for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The ``get`` sub-command gets the current Autoclass configuration for a bucket. The returned configuration has the following fields: ``enabled``: a boolean field indicating whether the feature is on or off. ``toggleTime``: a timestamp indicating when the enabled field was set. """ _DESCRIPTION = """ The `Autoclass <https://cloud.google.com/storage/docs/autoclass>`_ feature automatically selects the best storage class for objects based on access patterns. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("gs://{}:", name),' + ' format(" Enabled: {}",' + 'autoclass.enabled.yesno(True, False)),' + ' format(" Toggle Time: {}",' + 'autoclass.toggleTime))') class AutoclassCommand(Command): """Implements the gsutil autoclass command.""" command_spec = Command.CreateCommandSpec( 'autoclass', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='autoclass', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Autoclass feature', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--enable-autoclass', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-enable-autoclass', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _get_autoclass(self, blr): """Gets the autoclass setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['autoclass'], provider=bucket_url.scheme) bucket = str(bucket_url).rstrip('/') if bucket_metadata.autoclass: enabled = getattr(bucket_metadata.autoclass, 'enabled', False) toggle_time = getattr(bucket_metadata.autoclass, 'toggleTime', None) else: enabled = False toggle_time = None print('{}:\n' ' Enabled: {}\n' ' Toggle Time: {}'.format(bucket, enabled, toggle_time)) def _set_autoclass(self, blr, setting_arg): """Turns autoclass on or off for a bucket.""" bucket_url = blr.storage_url autoclass_config = apitools_messages.Bucket.AutoclassValue() autoclass_config.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(autoclass=autoclass_config) print('Setting Autoclass %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['autoclass'], provider=bucket_url.scheme) return 0 def _autoclass(self): """Handles autoclass command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('autoclass only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._get_autoclass elif subcommand == 'set': subcommand_func = self._set_autoclass setting_arg = self.args.pop(0) text_util.InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the autoclass command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._autoclass() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) requesterpays.py 0000644 00000020435 15025166704 0010044 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of requesterpays configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil requesterpays set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil requesterpays get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable Requester Pays for the specified bucket. """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the Requester Pays configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Requester Pays <https://cloud.google.com/storage/docs/requester-pays>`_ feature enables you to configure a Google Cloud Storage bucket so that the requester pays all costs related to accessing the bucket and its objects. The gsutil requesterpays command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=": "](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' + 'requester_pays.yesno("Enabled", "Disabled"))') class RequesterPaysCommand(Command): """Implementation of gsutil requesterpays command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'requesterpays', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ # ApiSelector.XML, # TODO: Uncomment once boto changes are added. ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='requesterpays', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Enable or disable requester pays for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--requester-pays' ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-requester-pays', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetRequesterPays(self): """Gets requesterpays configuration for a bucket.""" requesterpays_arg = self.args[0].lower() if requesterpays_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( billing=apitools_messages.Bucket.BillingValue()) if requesterpays_arg == 'on': self.logger.info('Enabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = True else: self.logger.info('Disabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetRequesterPays(self): """Gets requesterpays configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['billing']) for blr in bucket_iter: some_matched = True if blr.root_object.billing and blr.root_object.billing.requesterPays: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Disabled' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the requesterpays command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetRequesterPays metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetRequesterPays requesterpays_arg = self.args[0].lower() if requesterpays_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, requesterpays_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0 lifecycle.py 0000644 00000021325 15025166704 0007066 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of lifecycle configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LifecycleTranslation _GET_SYNOPSIS = """ gsutil lifecycle get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil lifecycle set <config-json-file> gs://<bucket_name>... """ _SYNOPSIS = _GET_SYNOPSIS + _SET_SYNOPSIS.lstrip('\n') + '\n' _GET_DESCRIPTION = """ <B>GET</B> Gets the lifecycle management configuration for a given bucket. You can get the lifecycle management configuration for only one bucket at a time. To update the configuration, you can redirect the output of the ``get`` command into a file, edit the file, and then set it on the bucket using the ``set`` sub-command. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the lifecycle management configuration on one or more buckets. The ``config-json-file`` specified on the command line should be a path to a local file containing the lifecycle configuration JSON document. """ _DESCRIPTION = """ You can use the ``lifecycle`` command to get or set lifecycle management policies for a given bucket. This command is supported for buckets only, not objects. For more information, see `Object Lifecycle Management <https://cloud.google.com/storage/docs/lifecycle>`_. The ``lifecycle`` command has two sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + """ <B>EXAMPLES</B> The following lifecycle management configuration JSON document specifies that all objects in this bucket that are more than 365 days old are deleted automatically: { "rule": [ { "action": {"type": "Delete"}, "condition": {"age": 365} } ] } The following empty lifecycle management configuration JSON document removes all lifecycle configuration for a bucket: {} """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LifecycleCommand(Command): """Implementation of gsutil lifecycle command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'lifecycle', command_name_aliases=['lifecycleconfig'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.JSON, ApiSelector.XML, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='lifecycle', help_name_aliases=[ 'getlifecycle', 'setlifecycle', ], help_type='command_help', help_one_line_summary=('Get or set lifecycle configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lifecycle-file={}'.format(self.args[1]), ] + self.args[2:], # Adds target bucket URLs. flag_map={}, ), }, flag_map={}, ) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=lifecycle_config,empty=\' ' 'has no lifecycle configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _SetLifecycleConfig(self): """Sets lifecycle configuration for a Google Cloud Storage bucket.""" lifecycle_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'lifecycle set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. lifecycle_file = open(lifecycle_arg, 'r') lifecycle_txt = lifecycle_file.read() lifecycle_file.close() # Iterate over URLs, expanding wildcards and setting the lifecycle on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['lifecycle']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting lifecycle configuration on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetLifecycle(lifecycle_txt, url, provider=url.scheme) else: lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt) bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetLifecycleConfig(self): """Gets lifecycle configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['lifecycle']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetLifecycle( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule: sys.stdout.write( LifecycleTranslation.JsonLifecycleFromMessage( bucket_metadata.lifecycle)) else: sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the lifecycle command.""" subcommand = self.args.pop(0) if subcommand == 'get': metrics.LogCommandParams(subcommands=[subcommand]) return self._GetLifecycleConfig() elif subcommand == 'set': metrics.LogCommandParams(subcommands=[subcommand]) return self._SetLifecycleConfig() else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (subcommand, self.command_name)) update.py 0000644 00000043353 15025166704 0006416 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of update command for updating gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil import signal import stat import sys import tarfile import tempfile import textwrap from six.moves import input import gslib from gslib.command import Command from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import system_util from gslib.utils.boto_util import GetConfigFilePaths from gslib.utils.boto_util import CERTIFICATE_VALIDATION_ENABLED from gslib.utils.constants import RELEASE_NOTES_URL from gslib.utils.text_util import CompareVersions from gslib.utils.update_util import DisallowUpdateIfDataInGsutilDir from gslib.utils.update_util import LookUpGsutilVersion from gslib.utils.update_util import GsutilPubTarball _SYNOPSIS = """ gsutil update [-f] [-n] [url] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: This command is not available if you're using a gsutil installation from a package manager or the Cloud SDK. When using the Cloud SDK, use ``gcloud components update``. The gsutil update command downloads the latest gsutil release, checks its version, and offers to let you update to it if it differs from the version you're currently running. Once you say "Y" to the prompt of whether to install the update, the gsutil update command locates where the running copy of gsutil is installed, unpacks the new version into an adjacent directory, moves the previous version aside, moves the new version to where the previous version was installed, and removes the moved-aside old version. Because of this, users are cautioned not to store data in the gsutil directory, since that data will be lost when you update gsutil. (Some users change directories into the gsutil directory to run the command. We advise against doing that, for this reason.) Note also that the gsutil update command will refuse to run if it finds user data in the gsutil directory. By default gsutil update will retrieve the new code from %s, but you can optionally specify a URL to use instead. This is primarily used for distributing pre-release versions of the code to a small group of early test users. NOTE: gsutil periodically checks whether a more recent software update is available. By default this check is performed every 30 days; you can change (or disable) this check by editing the software_update_check_period variable in the .boto config file. Note also that gsutil will only check for software updates if stdin, stdout, and stderr are all connected to a TTY, to avoid interfering with cron jobs, streaming transfers, and other cases where gsutil input or output are redirected from/to files or pipes. Software update periodic checks are also disabled by the gsutil -q option (see 'gsutil help options') <B>OPTIONS</B> -f Forces the update command to offer to let you update, even if you have the most current copy already. This can be useful if you have a corrupted local copy. -n Causes update command to run without prompting [Y/n] whether to continue if an update is available. """ % GsutilPubTarball()) class UpdateCommand(Command): """Implementation of gsutil update command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'update', command_name_aliases=['refresh'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='fn', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='update', help_name_aliases=['refresh'], help_type='command_help', help_one_line_summary='Update to the latest gsutil release', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _ExplainIfSudoNeeded(self, tf, dirs_to_remove, old_cwd): """Explains what to do if sudo needed to update gsutil software. Happens if gsutil was previously installed by a different user (typically if someone originally installed in a shared file system location, using sudo). Args: tf: Opened TarFile. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to if sudo is needed. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. Raises: CommandException: if errors encountered. """ # If running under Windows or Cygwin we don't need (or have) sudo. if system_util.IS_CYGWIN or system_util.IS_WINDOWS: return user_id = os.getuid() if os.stat(gslib.GSUTIL_DIR).st_uid == user_id: return # Won't fail - this command runs after main startup code that insists on # having a config file. config_file_list = GetConfigFilePaths() config_files = ' '.join(config_file_list) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) # Pick current protection of each boto config file for command that restores # protection (rather than fixing at 600) to support use cases like how GCE # installs a service account with an /etc/boto.cfg file protected to 644. chmod_cmds = [] for config_file in config_file_list: mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE]))) chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file)) raise CommandException('\n'.join( textwrap.wrap( 'Since it was installed by a different user previously, you will need ' 'to update using the following commands. You will be prompted for your ' 'password, and the install will run as "root". If you\'re unsure what ' 'this means please ask your system administrator for help:')) + ( '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update' '%s') % (config_files, config_files, self.gsutil_path, ' '.join(chmod_cmds)), informational=True) # This list is checked during gsutil update by doing a lowercased # slash-left-stripped check. For example "/Dev" would match the "dev" entry. unsafe_update_dirs = [ 'applications', 'auto', 'bin', 'boot', 'desktop', 'dev', 'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib', 'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net', 'null', 'network', 'opt', 'private', 'proc', 'program files', 'python', 'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr', 'var', 'volumes', 'win', 'win32', 'windows', 'winnt', ] def _EnsureDirsSafeForUpdate(self, dirs): """Raises Exception if any of dirs is known to be unsafe for gsutil update. This provides a fail-safe check to ensure we don't try to overwrite or delete any important directories. (That shouldn't happen given the way we construct tmp dirs, etc., but since the gsutil update cleanup uses shutil.rmtree() it's prudent to add extra checks.) Args: dirs: List of directories to check. Raises: CommandException: If unsafe directory encountered. """ for d in dirs: if not d: d = 'null' if d.lstrip(os.sep).lower() in self.unsafe_update_dirs: raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe ' 'directory (%s); aborting update' % d) def _CleanUpUpdateCommand(self, tf, dirs_to_remove, old_cwd): """Cleans up temp files etc. from running update command. Args: tf: Opened TarFile, or None if none currently open. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. """ if tf: tf.close() self._EnsureDirsSafeForUpdate(dirs_to_remove) for directory in dirs_to_remove: try: shutil.rmtree(directory) except OSError: # Ignore errors while attempting to remove old dirs under Windows. They # happen because of Windows exclusive file locking, and the update # actually succeeds but just leaves the old versions around in the # user's temp dir. if not system_util.IS_WINDOWS: raise if old_cwd: try: os.chdir(old_cwd) except OSError: pass def RunCommand(self): """Command entry point for the update command.""" if gslib.IS_PACKAGE_INSTALL: raise CommandException( 'The update command is only available for gsutil installed from a ' 'tarball. If you installed gsutil via another method, use the same ' 'method to update it.') if system_util.InvokedViaCloudSdk(): raise CommandException( 'The update command is disabled for Cloud SDK installs. Please run ' '"gcloud components update" to update it. Note: the Cloud SDK ' 'incorporates updates to the underlying tools approximately every 2 ' 'weeks, so if you are attempting to update to a recently created ' 'release / pre-release of gsutil it may not yet be available via ' 'the Cloud SDK.') https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED if not https_validate_certificates: raise CommandException( 'Your boto configuration has https_validate_certificates = False.\n' 'The update command cannot be run this way, for security reasons.') DisallowUpdateIfDataInGsutilDir() force_update = False no_prompt = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': force_update = True if o == '-n': no_prompt = True dirs_to_remove = [] tmp_dir = tempfile.mkdtemp() dirs_to_remove.append(tmp_dir) old_cwd = os.getcwd() os.chdir(tmp_dir) if not no_prompt: self.logger.info('Checking for software update...') if self.args: update_from_url_str = self.args[0] if not update_from_url_str.endswith('.tar.gz'): raise CommandException( 'The update command only works with tar.gz files.') for i, result in enumerate(self.WildcardIterator(update_from_url_str)): if i > 0: raise CommandException( 'Invalid update URL. Must name a single .tar.gz file.') storage_url = result.storage_url if storage_url.IsFileUrl() and not storage_url.IsDirectory(): if not force_update: raise CommandException( ('"update" command does not support "file://" URLs without the ' '-f option.')) elif not (storage_url.IsCloudUrl() and storage_url.IsObject()): raise CommandException( 'Invalid update object URL. Must name a single .tar.gz file.') else: update_from_url_str = GsutilPubTarball() # Try to retrieve version info from tarball metadata; failing that; download # the tarball and extract the VERSION file. The version lookup will fail # when running the update system test, because it retrieves the tarball from # a temp file rather than a cloud URL (files lack the version metadata). tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str) if tarball_version: tf = None else: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) tf.extractall() with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file: tarball_version = ver_file.read().strip() if not force_update and gslib.VERSION == tarball_version: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) if self.args: raise CommandException('You already have %s installed.' % update_from_url_str, informational=True) else: raise CommandException( 'You already have the latest gsutil release ' 'installed.', informational=True) if not no_prompt: CheckAndMaybePromptForAnalyticsEnabling() (_, major) = CompareVersions(tarball_version, gslib.VERSION) if major: print(('\n'.join( textwrap.wrap( 'This command will update to the "%s" version of gsutil at %s. ' 'NOTE: This a major new version, so it is strongly recommended ' 'that you review the release note details at %s before updating to ' 'this version, especially if you use gsutil in scripts.' % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL))))) else: print(('This command will update to the "%s" version of\ngsutil at %s' % (tarball_version, gslib.GSUTIL_DIR))) self._ExplainIfSudoNeeded(tf, dirs_to_remove, old_cwd) if no_prompt: answer = 'y' else: answer = input('Proceed? [y/N] ') if not answer or answer.lower()[0] != 'y': self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Not running update.', informational=True) if not tf: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) # Ignore keyboard interrupts during the update to reduce the chance someone # hitting ^C leaves gsutil in a broken state. RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN) # gslib.GSUTIL_DIR lists the path where the code should end up (like # /usr/local/gsutil), which is one level down from the relative path in the # tarball (since the latter creates files in ./gsutil). So, we need to # extract at the parent directory level. gsutil_bin_parent_dir = os.path.normpath( os.path.join(gslib.GSUTIL_DIR, '..')) # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR. old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) dirs_to_remove.append(old_dir) dirs_to_remove.append(new_dir) self._EnsureDirsSafeForUpdate(dirs_to_remove) try: tf.extractall(path=new_dir) except Exception as e: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Update failed: %s.' % e) # For enterprise mode (shared/central) installation, users with # different user/group than the installation user/group must be # able to run gsutil so we need to do some permissions adjustments # here. Since enterprise mode is not not supported for Windows # users, we can skip this step when running on Windows, which # avoids the problem that Windows has no find or xargs command. if not system_util.IS_WINDOWS: # Make all files and dirs in updated area owner-RW and world-R, and make # all directories owner-RWX and world-RX. for dirname, subdirs, filenames in os.walk(new_dir): for filename in filenames: fd = os.open(os.path.join(dirname, filename), os.O_RDONLY) os.fchmod(fd, stat.S_IWRITE | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) os.close(fd) for subdir in subdirs: fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRGRP | stat.S_IROTH) os.close(fd) # Make main gsutil script owner-RWX and world-RX. fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) os.close(fd) # Move old installation aside and new into place. os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old')) os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL) self.logger.info('Update complete.') return 0 def _FetchAndOpenGsutilTarball(self, update_from_url_str): self.command_runner.RunNamedCommand( 'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers, self.debug, skip_update_check=True, ) # Note: tf is closed in _CleanUpUpdateCommand. tf = tarfile.open('gsutil.tar.gz') tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. return tf acl.py 0000644 00000057443 15025166704 0005700 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of acl command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from apitools.base.py import encoding from gslib import metrics from gslib import gcs_json_api from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.storage_url import RaiseErrorIfUrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil acl set [-f] [-r] [-a] (<file-path>|<predefined-acl>) url... """ _GET_SYNOPSIS = """ gsutil acl get url """ _CH_SYNOPSIS = """ gsutil acl ch [-f] [-r] <grant>... url... where each <grant> is one of the following forms: -u <id>|<email>:<permission> -g <id>|<email>|<domain>|All|AllAuth:<permission> -p (viewers|editors|owners)-<project number>:<permission> -d <id>|<email>|<domain>|All|AllAuth|(viewers|editors|owners)-<project number> """ _GET_DESCRIPTION = """ <B>GET</B> The ``acl get`` command gets the ACL text for a bucket or object, which you can save and edit for the acl set command. """ _SET_DESCRIPTION = """ <B>SET</B> The ``acl set`` command allows you to set an Access Control List on one or more buckets and objects. As part of the command, you must specify either a predefined ACL or the path to a file that contains ACL text. The simplest way to use the ``acl set`` command is to specify one of the predefined ACLs, e.g.,: gsutil acl set private gs://example-bucket/example-object If you want to make an object or bucket publicly readable or writable, it is recommended to use ``acl ch``, to avoid accidentally removing OWNER permissions. See the ``acl ch`` section for details. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. If you want to define more fine-grained control over your data, you can retrieve an ACL using the ``acl get`` command, save the output to a file, edit the file, and then use the ``acl set`` command to set that ACL on the buckets and/or objects. For example: gsutil acl get gs://bucket/file.txt > acl.txt Make changes to acl.txt such as adding an additional grant, then: gsutil acl set acl.txt gs://cats/file.txt Note that you can set an ACL on multiple buckets or objects at once. For example, to set ACLs on all .jpg files found in a bucket: gsutil acl set acl.txt gs://bucket/**.jpg If you have a large number of ACLs to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m acl set acl.txt gs://bucket/**.jpg Note that multi-threading/multi-processing is only done when the named URLs refer to objects, which happens either if you name specific objects or if you enumerate objects by using an object wildcard or specifying the acl -r flag. <B>SET OPTIONS</B> The "set" sub-command has the following options -R, -r Performs "acl set" request recursively, to all objects under the specified URL. -a Performs "acl set" request on all object versions. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. If some of the ACLs couldn't be set, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m acl...". """ _CH_DESCRIPTION = """ <B>CH</B> The "acl ch" (or "acl change") command updates access control lists, similar in spirit to the Linux chmod command. You can specify multiple access grant additions and deletions in a single command run; all changes will be made atomically to each object in turn. For example, if the command requests deleting one grant and adding a different grant, the ACLs being updated will never be left in an intermediate state where one grant has been deleted but the second grant not yet added. Each change specifies a user or group grant to add or delete, and for grant additions, one of R, W, O (for the permission to be granted). A more formal description is provided in a later section; below we provide examples. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Grant anyone on the internet READ access to the object example-object: gsutil acl ch -u allUsers:R gs://example-bucket/example-object NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Grant the user john.doe@example.com READ access to all objects in example-bucket that begin with folder/: gsutil acl ch -r -u john.doe@example.com:R gs://example-bucket/folder/ Grant the group admins@example.com OWNER access to all jpg files in example-bucket: gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg Grant the owners of project example-project WRITE access to the bucket example-bucket: gsutil acl ch -p owners-example-project:W gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. Remove access to the bucket example-bucket for the viewers of project number 12345: gsutil acl ch -d viewers-12345 gs://example-bucket NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in the given project. Attempts to do so will appear to succeed, but the service will add the project owners group into the new set of ACLs before applying it. Note that removing a project requires you to reference the project by its number (which you can see with the acl get command) as opposed to its project ID string. Grant the service account foo@developer.gserviceaccount.com WRITE access to the bucket example-bucket: gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket Grant all users from the `G Suite <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ access to the bucket gcs.my-domain.org: gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org Remove any current access by john.doe@example.com from the bucket example-bucket: gsutil acl ch -d john.doe@example.com gs://example-bucket If you have a large number of objects to update, enabling multi-threading with the gsutil -m flag can significantly improve performance. The following command adds OWNER for admin@example.org using multi-threading: gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket Grant READ access to everyone from my-domain.org and to all authenticated users, and grant OWNER to admin@mydomain.org, for the buckets my-bucket and my-other-bucket, with multi-threading enabled: gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \\ -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket <B>CH ROLES</B> You may specify the following roles with either their shorthand or their full name: R: READ W: WRITE O: OWNER For more information on these roles and the access they grant, see the permissions section of the `Access Control Lists page <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. <B>CH ENTITIES</B> There are four different entity types: Users, Groups, All Authenticated Users, and All Users. Users are added with -u and a plain ID or email address, as in "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. Groups are like users, but specified with the -g flag, as in "-g power-users@example.com:O". Groups may also be specified as a full domain, as in "-g my-company.com:r". allAuthenticatedUsers and allUsers are specified directly, as in "-g allUsers:R" or "-g allAuthenticatedUsers:O". These are case insensitive, and may be shortened to "all" and "allauth", respectively. Removing roles is specified with the -d flag and an ID, email address, domain, or one of allUsers or allAuthenticatedUsers. Many entities' roles can be specified on the same command line, allowing bundled changes to be executed in a single run. This will reduce the number of requests made to the server. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -R, -r Performs acl ch request recursively, to all objects under the specified URL. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = (""" The acl command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION])) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) def _ApplyExceptionHandler(cls, exception): cls.logger.error('Encountered a problem: %s', exception) cls.everything_set_okay = False def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None): cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state) class AclCommand(Command): """Implementation of gsutil acl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'acl', command_name_aliases=['getacl', 'setacl', 'chacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument() ], 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='acl', help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'], help_type='command_help', help_one_line_summary='Get, set, or change bucket and/or object ACLs', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text }, ) def _get_shim_command_group(self): object_or_bucket_urls = [StorageUrlFromString(url) for url in self.args] recurse = False for (flag_key, _) in self.sub_opts: if flag_key in ('-r', '-R'): recurse = True break RaiseErrorIfUrlsAreMixOfBucketsAndObjects(object_or_bucket_urls, recurse) if object_or_bucket_urls[0].IsBucket() and not recurse: return 'buckets' else: return 'objects' def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'describe', '--format=multi(acl:format=json)' ], flag_map={}) elif sub_command == 'set': # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() acl_file_or_predefined_acl = self.args.pop(0) if os.path.isfile(acl_file_or_predefined_acl): acl_flag = '--acl-file=' + acl_file_or_predefined_acl else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl acl_flag = '--predefined-acl=' + predefined_acl command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'] + [acl_flag], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'], flag_map={ '-g': GcloudStorageFlag('--add-acl-grant'), '-p': GcloudStorageFlag('--add-acl-grant'), '-u': GcloudStorageFlag('--add-acl-grant'), '-d': GcloudStorageFlag('--remove-acl-grant'), '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'): return 1 else: return 0 def _SetAcl(self): """Parses options and sets ACLs on the specified buckets/objects.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException as unused_e: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _ChAcl(self): """Parses options and changes ACLs on the specified buckets/objects.""" self.parse_versions = True self.changes = [] self.continue_on_error = False if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-g': if 'gserviceaccount.com' in a: raise CommandException( 'Service accounts are considered users, not groups; please use ' '"gsutil acl ch -u" instead of "gsutil acl ch -g"') self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) elif o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) elif o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) elif o == '-d': self.changes.append(acl_helper.AclDel(a)) elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) self.everything_set_okay = True self.ApplyAclFunc(_ApplyAclChangesWrapper, _ApplyExceptionHandler, self.args, object_fields=['acl', 'generation', 'metageneration']) if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _RaiseForAccessDenied(self, url): self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, name_expansion_result, thread_state=None): """Applies the changes in self.changes to the provided URL. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, gsutil Cloud API instance to apply the changes. """ if thread_state: gsutil_api = thread_state else: gsutil_api = self.gsutil_api url = name_expansion_result.expanded_storage_url if url.IsBucket(): bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme, fields=['acl', 'metageneration']) current_acl = bucket.acl elif url.IsObject(): gcs_object = encoding.JsonToMessage(apitools_messages.Object, name_expansion_result.expanded_result) current_acl = gcs_object.acl if not current_acl: self._RaiseForAccessDenied(url) if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return try: if url.IsBucket(): preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(acl=current_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) else: # Object preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) object_metadata = apitools_messages.Object(acl=current_acl) try: gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=url.generation, fields=['id']) except PreconditionException as e: # Special retry case where we want to do an additional step, the read # of the read-modify-write cycle, to fetch the correct object # metadata before reattempting ACL changes. self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api) self.logger.info('Updated ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._RaiseForAccessDenied(url) except PreconditionException as e: # For objects, retry attempts should have already been handled. if url.IsObject(): raise CommandException(str(e)) # For buckets, raise PreconditionException and continue to next retry. raise e @Retry(PreconditionException, tries=3, timeout_secs=1) def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api): """Reattempts object ACL changes after a PreconditionException.""" gcs_object = gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=url.scheme, fields=['acl', 'generation', 'metageneration']) current_acl = gcs_object.acl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return object_metadata = apitools_messages.Object(acl=current_acl) preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=gcs_object.generation, fields=['id']) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, acl_message, 'acl', self.logger) return modification_count def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self.GetAndPrintAcl(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetAcl() elif action_subcommand in ('ch', 'change'): metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChAcl() else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help acl".') % (action_subcommand, self.command_name)) return 0 mb.py 0000644 00000041153 15025166704 0005526 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of mb command for creating cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import textwrap from gslib.cloud_api import AccessDeniedException, BadRequestException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.rpo import VALID_RPO_VALUES from gslib.commands.rpo import VALID_RPO_VALUES_STRING from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils.text_util import InsistOnOrOff from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.encryption_helper import ValidateCMEK _SYNOPSIS = """ gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>] [--autoclass] [--retention <time>] [--pap <setting>] [--placement <region1>,<region2>] [--rpo {}] gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Create one or more new buckets. Google Cloud Storage has a single namespace, so you are not allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "gsutil help naming"). If you don't specify a project ID or project number using the -p option, the buckets are created using the default project ID specified in your `gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. The -l option specifies the location for the buckets. Once a bucket is created in a given location, it cannot be moved to a different location. Instead, you need to create a new bucket, move the data over, and then delete the original bucket. <B>BUCKET STORAGE CLASSES</B> You can specify one of the `storage classes <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket with the -c option. Example: gsutil mb -c nearline gs://some-bucket See online documentation for `pricing <https://cloud.google.com/storage/pricing>`_ and `SLA <https://cloud.google.com/storage/sla>`_ details. If you don't specify a -c option, the bucket is created with the default storage class Standard Storage. <B>BUCKET LOCATIONS</B> You can specify one of the `available locations <https://cloud.google.com/storage/docs/locations>`_ for a bucket with the -l option. Examples: gsutil mb -l asia gs://some-bucket gsutil mb -c standard -l us-east1 gs://some-bucket If you don't specify a -l option, the bucket is created in the default location (US). <B>Retention Policy</B> You can specify retention period in one of the following formats: --retention <number>s Specifies retention period of <number> seconds for objects in this bucket. --retention <number>d Specifies retention period of <number> days for objects in this bucket. --retention <number>m Specifies retention period of <number> months for objects in this bucket. --retention <number>y Specifies retention period of <number> years for objects in this bucket. Examples: gsutil mb --retention 1y gs://some-bucket gsutil mb --retention 36m gs://some-bucket If you don't specify a --retention option, the bucket is created with no retention policy. <B>OPTIONS</B> --autoclass Enables the Autoclass feature that automatically sets object storage classes. -b <on|off> Specifies the uniform bucket-level access setting. When "on", ACLs assigned to objects in the bucket are not evaluated. Consequently, only IAM policies grant access to objects in these buckets. Default is "off". -c class Specifies the default storage class. Default is ``Standard``. See `Available storage classes <https://cloud.google.com/storage/docs/storage-classes#classes>`_ for a list of possible values. -k <key> Set the default KMS key using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -l location Can be any supported location. See https://cloud.google.com/storage/docs/locations for a discussion of this distinction. Default is US. Locations are case insensitive. -p project Specifies the project ID or project number to create the bucket under. -s class Same as -c. --retention time Specifies the retention policy. Default is no retention policy. This can only be set on gs:// buckets and requires using the JSON API. For more details about retention policy see "gsutil help retention" --pap setting Specifies the public access prevention setting. Valid values are "enforced" or "inherited". When "enforced", objects in this bucket cannot be made publicly accessible. Default is "inherited". --placement reg1,reg2 Two regions that form the custom dual-region. Only regions within the same continent are or will ever be valid. Invalid location pairs (such as mixed-continent, or with unsupported regions) will return an error. --rpo setting Specifies the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_. This flag is not valid for single-region buckets, and multi-region buckets only accept a value of DEFAULT. Valid values for dual region buckets are {rpo_values}. If unspecified, DEFAULT is applied for dual-region and multi-region buckets. """.format(rpo_values=VALID_RPO_VALUES_STRING)) # Regex to disallow buckets violating charset or not [3..255] chars total. BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$') # Regex to disallow buckets with individual DNS labels longer than 63. TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}') _RETENTION_FLAG = '--retention' IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class MbCommand(Command): """Implementation of gsutil mb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mb', command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='b:c:l:p:s:k:', supported_private_args=[ 'autoclass', 'retention=', 'pap=', 'placement=', 'rpo=' ], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mb', help_name_aliases=[ 'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra', 'dras', 'reduced_availability', 'durable_reduced_availability', 'rr', 'reduced_redundancy', 'standard', 'storage class', 'nearline', 'nl', ], help_type='command_help', help_one_line_summary='Make buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'create'], flag_map={ '-b': GcloudStorageFlag({ 'on': '--uniform-bucket-level-access', 'off': None, }), '-c': GcloudStorageFlag('--default-storage-class'), '-k': GcloudStorageFlag('--default-encryption-key'), '-l': GcloudStorageFlag('--location'), '-p': GcloudStorageFlag('--project'), '--pap': GcloudStorageFlag({ 'enforced': '--public-access-prevention', 'inherited': None, }), '--placement': GcloudStorageFlag('--placement'), _RETENTION_FLAG: GcloudStorageFlag('--retention-period'), '--rpo': GcloudStorageFlag('--recovery-point-objective') }, ) def get_gcloud_storage_args(self): retention_arg_idx = 0 while retention_arg_idx < len(self.sub_opts): if self.sub_opts[retention_arg_idx][0] == _RETENTION_FLAG: break retention_arg_idx += 1 if retention_arg_idx < len(self.sub_opts): # Convert retention time to seconds, which gcloud knows how to handle. self.sub_opts[retention_arg_idx] = ( _RETENTION_FLAG, str(RetentionInSeconds(self.sub_opts[retention_arg_idx][1])) + 's') return super().get_gcloud_storage_args(MbCommand.gcloud_storage_map) def RunCommand(self): """Command entry point for the mb command.""" autoclass = False bucket_policy_only = None kms_key = None location = None storage_class = None seconds = None placements = None public_access_prevention = None rpo = None json_only_flags_in_command = [] if self.sub_opts: for o, a in self.sub_opts: if o == '--autoclass': autoclass = True json_only_flags_in_command.append(o) elif o == '-k': kms_key = a ValidateCMEK(kms_key) json_only_flags_in_command.append(o) elif o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) elif o == _RETENTION_FLAG: seconds = RetentionInSeconds(a) elif o == '--rpo': rpo = a.strip() if rpo not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for --rpo. Must be one of: {},' ' provided: {}'.format(VALID_RPO_VALUES_STRING, a)) json_only_flags_in_command.append(o) elif o == '-b': InsistOnOrOff(a, 'Only on and off values allowed for -b option') bucket_policy_only = (a == 'on') json_only_flags_in_command.append(o) elif o == '--pap': public_access_prevention = a json_only_flags_in_command.append(o) elif o == '--placement': placements = a.split(',') if len(placements) != 2: raise CommandException( 'Please specify two regions separated by comma without space.' ' Specified: {}'.format(a)) json_only_flags_in_command.append(o) bucket_metadata = apitools_messages.Bucket(location=location, rpo=rpo, storageClass=storage_class) if autoclass: bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue( enabled=autoclass) if bucket_policy_only or public_access_prevention: bucket_metadata.iamConfiguration = IamConfigurationValue() iam_config = bucket_metadata.iamConfiguration if bucket_policy_only: iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = bucket_policy_only if public_access_prevention: iam_config.publicAccessPrevention = public_access_prevention if kms_key: encryption = apitools_messages.Bucket.EncryptionValue() encryption.defaultKmsKeyName = kms_key bucket_metadata.encryption = encryption if placements: placement_config = apitools_messages.Bucket.CustomPlacementConfigValue() placement_config.dataLocations = placements bucket_metadata.customPlacementConfig = placement_config for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if seconds is not None: if bucket_url.scheme != 'gs': raise CommandException('Retention policy can only be specified for ' 'GCS buckets.') retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) bucket_metadata.retentionPolicy = retention_policy if json_only_flags_in_command and self.gsutil_api.GetApiSelector( bucket_url.scheme) != ApiSelector.JSON: raise CommandException('The {} option(s) can only be used for GCS' ' Buckets with the JSON API'.format( ', '.join(json_only_flags_in_command))) if not bucket_url.IsBucket(): raise CommandException('The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except AccessDeniedException as e: message = e.reason if 'key' in message: # This will print the error reason and append the following as a # suggested next step: # # To authorize, run: # gsutil kms authorize \ # -k <kms_key> \ # -p <project_id> message += ' To authorize, run:\n gsutil kms authorize' message += ' \\\n -k %s' % kms_key if (self.project_id): message += ' \\\n -p %s' % self.project_id raise CommandException(message) else: raise except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0 retention.py 0000644 00000073414 15025166704 0007144 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Retention Policy configuration command for buckets.""" from __future__ import absolute_import import time from apitools.base.py import encoding from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.retention_util import ConfirmLockRequest from gslib.utils.retention_util import ReleaseEventHoldFuncWrapper from gslib.utils.retention_util import ReleaseTempHoldFuncWrapper from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.retention_util import RetentionPolicyToString from gslib.utils.retention_util import SetEventHoldFuncWrapper from gslib.utils.retention_util import SetTempHoldFuncWrapper from gslib.utils.retention_util import UpdateObjectMetadataExceptionHandler from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders _SET_SYNOPSIS = """ gsutil retention set <retention_period> gs://<bucket_name>... """ _CLEAR_SYNOPSIS = """ gsutil retention clear gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil retention get gs://<bucket_name>... """ _LOCK_SYNOPSIS = """ gsutil retention lock gs://<bucket_name>... """ _EVENT_DEFAULT_SYNOPSIS = """ gsutil retention event-default (set|release) gs://<bucket_name>... """ _EVENT_SYNOPSIS = """ gsutil retention event (set|release) gs://<bucket_name>/<object_name>... """ _TEMP_SYNOPSIS = """ gsutil retention temp (set|release) gs://<bucket_name>/<object_name>... """ _SET_DESCRIPTION = """ <B>SET</B> You can configure a data retention policy for a Cloud Storage bucket that governs how long objects in the bucket must be retained. You can also lock the data retention policy, permanently preventing the policy from being reduced or removed. For more information, see `Retention policies and Bucket Lock <https://cloud.google.com/storage/docs/bucket-lock>`_. The ``gsutil retention set`` command allows you to set or update the retention policy on one or more buckets. To remove an unlocked retention policy from one or more buckets, use the ``gsutil retention clear`` command. The ``set`` sub-command can set a retention policy with the following formats: <B>SET FORMATS</B> Formats for the ``set`` subcommand include: <number>s Specifies retention period of <number> seconds for objects in this bucket. <number>d Specifies retention period of <number> days for objects in this bucket. <number>m Specifies retention period of <number> months for objects in this bucket. <number>y Specifies retention period of <number> years for objects in this bucket. GCS JSON API accepts retention periods as number of seconds. Durations provided in terms of days, months or years are converted to their rough equivalent values in seconds, using the following conversions: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Retention periods must be greater than 0 and less than 100 years. Retention durations must be in only one form (seconds, days, months, or years), and not a combination of them. Note that while it is possible to specify retention durations shorter than a day (using seconds), enforcement of such retention periods is not guaranteed. Such durations may only be used for testing purposes. <B>EXAMPLES</B> Setting a retention policy with a duration of 1 year on a bucket: gsutil retention set 1y gs://my-bucket Setting a retention policy with a duration of 36 months on a bucket: gsutil retention set 36m gs://some-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the retention policy on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention set 1y gs://my-bucket """ _CLEAR_DESCRIPTION = """ <B>CLEAR</B> The ``gsutil retention clear`` command removes an unlocked retention policy from one or more buckets. You cannot remove or reduce the duration of a locked retention policy. <B>EXAMPLES</B> Clearing an unlocked retention policy from a bucket: gsutil retention clear gs://my-bucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil retention get`` command retrieves the retention policy for a given bucket and displays a human-readable representation of the configuration. """ _LOCK_DESCRIPTION = """ <B>LOCK</B> The ``gsutil retention lock`` command PERMANENTLY locks an unlocked retention policy on one or more buckets. CAUTION: A locked retention policy cannot be removed from a bucket or reduced in duration. Once locked, deleting the bucket is the only way to "remove" a retention policy. """ _EVENT_DEFAULT_DESCRIPTION = """ <B>EVENT-DEFAULT</B> The ``gsutil retention event-default`` command sets the default value for an event-based hold on one or more buckets. By setting the default event-based hold on a bucket, newly-created objects inherit that value as their event-based hold (it is not applied retroactively). <B>EXAMPLES</B> Setting the default event-based hold on a bucket: gsutil retention event-default set gs://my-bucket Releasing the default event-based hold on a bucket: gsutil retention event-default release gs://my-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the default event-based hold on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event-default set gs://my-bucket """ _EVENT_DESCRIPTION = """ <B>EVENT</B> The ``gsutil retention event`` command enables or disables an event-based hold on an object. <B>EXAMPLES</B> Setting the event-based hold on an object: gsutil retention event set gs://my-bucket/my-object Releasing the event-based hold on an object: gsutil retention event release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the event-based hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event set gs://my-bucket/my-object If you want to set or release an event-based hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets an event-based hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention event set gs://my-bucket/*.jpg """ _TEMP_DESCRIPTION = """ <B>TEMP</B> The ``gsutil retention temp`` command enables or disables a temporary hold on an object. <B>EXAMPLES</B> Setting the temporary hold on an object: gsutil retention temp set gs://my-bucket/my-object Releasing the temporary hold on an object: gsutil retention temp release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the temporary hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention temp set gs://my-bucket/my-object If you want to set or release a temporary hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets a temporary hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention temp set gs://bucket/*.jpg """ _SYNOPSIS = (_SET_SYNOPSIS + _CLEAR_SYNOPSIS + _GET_SYNOPSIS + _LOCK_SYNOPSIS + _EVENT_DEFAULT_SYNOPSIS + _EVENT_SYNOPSIS + _TEMP_SYNOPSIS) _DESCRIPTION = (_SET_DESCRIPTION + _CLEAR_DESCRIPTION + _GET_DESCRIPTION + _LOCK_DESCRIPTION + _EVENT_DEFAULT_DESCRIPTION + _EVENT_DESCRIPTION + _TEMP_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _clear_help_text = CreateHelpText(_CLEAR_SYNOPSIS, _CLEAR_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _lock_help_text = CreateHelpText(_LOCK_SYNOPSIS, _LOCK_DESCRIPTION) _event_default_help_text = CreateHelpText(_EVENT_DEFAULT_SYNOPSIS, _EVENT_DEFAULT_DESCRIPTION) _event_help_text = CreateHelpText(_EVENT_SYNOPSIS, _EVENT_DESCRIPTION) _temp_help_text = CreateHelpText(_TEMP_SYNOPSIS, _TEMP_DESCRIPTION) class RetentionCommand(Command): """Implementation of gsutil retention command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'retention', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'event-default': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'event': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'temp': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='retention', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Provides utilities to interact with Retention Policy feature.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'clear': _clear_help_text, 'lock': _lock_help_text, 'event-default': _event_default_help_text, 'event': _event_help_text, 'temp': _temp_help_text }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--retention-period={}s'.format( RetentionInSeconds(self.args[1])) ] + self.args[2:], # Adds target bucket URLs. flag_map={}), }, flag_map={}) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'clear': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-retention-period' ], flag_map={}, ), 'event': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'event-default': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-default-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'get': GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'describe', '--format=yaml(retentionPolicy)', '--raw' ], flag_map={}), 'lock': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lock-retention-period' ], flag_map={}, ), 'temp': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--temporary-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-temporary-hold' ], flag_map={}, ), }, flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the retention command.""" # If the only credential type the user supplies in their boto file is HMAC, # GetApiSelector logic will force us to use the XML API, which bucket lock # does not support at the moment. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException(('The {} command can only be used with the GCS ' 'JSON API. If you have only supplied hmac ' 'credentials in your boto file, please instead ' 'supply a credential type that can be used with ' 'the JSON API.').format(self.command_name)) self.preconditions = PreconditionsFromHeaders(self.headers) action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'set': func = self._SetRetention elif action_subcommand == 'clear': func = self._ClearRetention elif action_subcommand == 'get': func = self._GetRetention elif action_subcommand == 'lock': func = self._LockRetention elif action_subcommand == 'event-default': func = self._DefaultEventHold elif action_subcommand == 'event': func = self._EventHold elif action_subcommand == 'temp': func = self._TempHold else: raise CommandException( ('Invalid subcommand "{}" for the {} command.\n' 'See "gsutil help retention".').format(action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields, log_msg_template): preconditions = Preconditions( meta_gen_match=self.preconditions.meta_gen_match) # Iterate over URLs, expanding wildcards and setting the new bucket metadata # on each bucket. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info(log_msg_template, blr) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata_update, preconditions=preconditions, provider=url.scheme, fields=fields) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def ObjectUpdateMetadataFunc(self, patch_obj_metadata, log_template, name_expansion_result, thread_state=None): """Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info(log_template, exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetObjectNameExpansionIterator(self, url_args): return NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metageneration']) def _GetSeekAheadNameExpansionIterator(self, url_args): return SeekAheadNameExpansionIterator(self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) def _SetRetention(self): """Set retention retention_period on one or more buckets.""" seconds = RetentionInSeconds(self.args[0]) retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) log_msg_template = 'Setting Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _ClearRetention(self): """Clear retention retention_period on one or more buckets.""" retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=None)) log_msg_template = 'Clearing Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _GetRetention(self): """Get Retention Policy for a single bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['retentionPolicy']) print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url)) return 0 def _LockRetention(self): """Lock Retention Policy on one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and setting the Retention Policy # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True # Get bucket metadata to provide a precondition. bucket_metadata = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['id', 'metageneration', 'retentionPolicy']) if (not (bucket_metadata.retentionPolicy and bucket_metadata.retentionPolicy.retentionPeriod)): # TODO: implement '-c' flag to continue_on_error raise CommandException( 'Bucket "{}" does not have an Unlocked Retention Policy.'.format( url.bucket_name)) elif bucket_metadata.retentionPolicy.isLocked is True: self.logger.error('Retention Policy on "%s" is already locked.', blr) elif ConfirmLockRequest(url.bucket_name, bucket_metadata.retentionPolicy): self.logger.info('Locking Retention Policy on %s...', blr) self.gsutil_api.LockRetentionPolicy(url.bucket_name, bucket_metadata.metageneration, provider=url.scheme) else: self.logger.error( ' Abort Locking Retention Policy on {}'.format(blr)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _DefaultEventHold(self): """Sets default value for Event-Based Hold on one or more buckets.""" hold = None if self.args: if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention event-default"' ' command.\nSee "gsutil help retention event".').format( self.sub_opts)) verb = 'Setting' if hold else 'Releasing' log_msg_template = '{} default Event-Based Hold on %s...'.format(verb) bucket_metadata_update = apitools_messages.Bucket( defaultEventBasedHold=hold) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'defaultEventBasedHold'], log_msg_template=log_msg_template) return 0 def _EventHold(self): """Sets or unsets Event-Based Hold on one or more objects.""" sub_command_name = 'event' sub_command_full_name = 'Event-Based' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetEventHoldFuncWrapper if hold else ReleaseEventHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _TempHold(self): """Sets or unsets Temporary Hold on one or more objects.""" sub_command_name = 'temp' sub_command_full_name = 'Temporary' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetTempHoldFuncWrapper if hold else ReleaseTempHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _ProcessHoldArgs(self, sub_command_name): """Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. """ hold = None if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention {}" command.\n' 'See "gsutil help retention {}".').format(self.args[0], sub_command_name, sub_command_name)) return hold def _SetHold(self, obj_metadata_update_wrapper, url_args, sub_command_full_name): """Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" """ if len(url_args) == 1 and not self.recursion_requested: url = StorageUrlFromString(url_args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL ({}) must name an object'.format( url_args[0])) name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args) seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True try: # TODO: implement '-c' flag to continue_on_error # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(obj_metadata_update_wrapper, name_expansion_iterator, UpdateObjectMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException( '{} Hold for some objects could not be set.'.format( sub_command_full_name)) cat.py 0000644 00000012671 15025166704 0005702 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import cat_helper from gslib.utils import constants from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap if six.PY3: long = int _SYNOPSIS = """ gsutil cat [-h] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The cat command outputs the contents of one or more URLs to stdout. While the cat command does not compute a checksum, it is otherwise equivalent to doing: gsutil cp url... - (The final '-' causes gsutil to stream the output to stdout.) WARNING: The gsutil cat command does not compute a checksum of the downloaded data. Therefore, we recommend that users either perform their own validation of the output of gsutil cat or use gsutil cp or rsync (both of which perform integrity checking automatically). <B>OPTIONS</B> -h Prints short header for each object. For example: gsutil cat -h gs://bucket/meeting_notes/2012_Feb/*.txt This would print a header with the object name before the contents of each text object that matched the wildcard. -r range Causes gsutil to output just the specified byte range of the object. Ranges can be of these forms: start-end (e.g., -r 256-5939) start- (e.g., -r 256-) -numbytes (e.g., -r -5) where offsets start at 0, start-end means to return bytes start through end (inclusive), start- means to return bytes start through the end of the object, and -numbytes means to return the last numbytes of the object. For example: gsutil cat -r 256-939 gs://bucket/object returns bytes 256 through 939, while: gsutil cat -r -5 gs://bucket/object returns the final 5 bytes of the object. """) class CatCommand(Command): """Implementation of gsutil cat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='hr:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Concatenate object content to stdout', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cat'], flag_map={ '-h': GcloudStorageFlag('-d'), '-r': GcloudStorageFlag('-r'), }, ) # Command entry point. def RunCommand(self): """Command entry point for the cat command.""" show_header = False request_range = None start_byte = 0 end_byte = None if self.sub_opts: for o, a in self.sub_opts: if o == '-h': show_header = True elif o == '-r': request_range = a.strip() # This if statement ensures the full object is returned # instead of throwing a CommandException. if request_range == '-': continue range_matcher = re.compile( '^(?P<start>[0-9]+)-(?P<end>[0-9]*)$|^(?P<endslice>-[0-9]+)$') range_match = range_matcher.match(request_range) if not range_match: raise CommandException('Invalid range (%s)' % request_range) if range_match.group('start'): start_byte = long(range_match.group('start')) if range_match.group('end'): end_byte = long(range_match.group('end')) if range_match.group('endslice'): start_byte = long(range_match.group('endslice')) else: self.RaiseInvalidArgumentException() return cat_helper.CatHelper(self).CatUrlStrings(self.args, show_header=show_header, start_byte=start_byte, end_byte=end_byte) label.py 0000644 00000037534 15025166704 0006217 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of label command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import codecs import json import os import six from gslib import metrics from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LabelTranslation _SET_SYNOPSIS = """ gsutil label set <label-json-file> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil label get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil label ch <label_modifier>... gs://<bucket_name>... where each <label_modifier> is one of the following forms: -l <key>:<value> -d <key> """ _GET_DESCRIPTION = """ <B>GET</B> The "label get" command gets the `labels <https://cloud.google.com/storage/docs/tags-and-labels#bucket-labels>`_ applied to a bucket, which you can save and edit for use with the "label set" command. """ _SET_DESCRIPTION = """ <B>SET</B> The "label set" command allows you to set the labels on one or more buckets. You can retrieve a bucket's labels using the "label get" command, save the output to a file, edit the file, and then use the "label set" command to apply those labels to the specified bucket(s). For example: gsutil label get gs://bucket > labels.json Make changes to labels.json, such as adding an additional label, then: gsutil label set labels.json gs://example-bucket Note that you can set these labels on multiple buckets at once: gsutil label set labels.json gs://bucket-foo gs://bucket-bar """ _CH_DESCRIPTION = """ <B>CH</B> The "label ch" command updates a bucket's label configuration, applying the label changes specified by the -l and -d flags. You can specify multiple label changes in a single command run; all changes will be made atomically to each bucket. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Add the label "key-foo:value-bar" to the bucket "example-bucket": gsutil label ch -l key-foo:value-bar gs://example-bucket Change the above label to have a new value: gsutil label ch -l key-foo:other-value gs://example-bucket Add a new label and delete the old one from above: gsutil label ch -l new-key:new-value -d key-foo gs://example-bucket <B>CH OPTIONS</B> The "ch" sub-command has the following options -l Add or update a label with the specified key and value. -d Remove the label with the specified key. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Gets, sets, or changes the label configuration (also called the tagging configuration by other storage providers) of one or more buckets. An example label JSON document looks like the following: { "your_label_key": "your_label_value", "your_other_label_key": "your_other_label_value" } The label command has three sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + _CH_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class LabelCommand(Command): """Implementation of gsutil label command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'label', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='l:d:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='label', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change the label configuration of a bucket.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=labels,empty=\' has no label ' 'configuration.\',empty_prefix_key=storage_url,indent=2]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--labels-file'], flag_map={}, ), 'ch': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-d': GcloudStorageFlag( '--remove-labels', repeat_type=shim_util.RepeatFlagType.LIST), '-l': GcloudStorageFlag( '--update-labels', repeat_type=shim_util.RepeatFlagType.DICT), }, ), }, flag_map={}) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 # Filename comes before bucket arg(s). return 1 def _SetLabel(self): """Parses options and sets labels on the specified buckets.""" # At this point, "set" has been popped off the front of self.args. if len(self.args) < 2: self.RaiseWrongNumberOfArgumentsException() label_filename = self.args[0] if not os.path.isfile(label_filename): raise CommandException('Could not find the file "%s".' % label_filename) with codecs.open(label_filename, 'r', UTF8) as label_file: label_text = label_file.read() @Retry(PreconditionException, tries=3, timeout_secs=1) def _SetLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) if url.scheme == 's3': # Uses only XML. self.gsutil_api.XmlPassThroughSetTagging(label_text, url, provider=url.scheme) else: # Must be a 'gs://' bucket. labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None new_label_json = json.loads(label_text) if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # Perform a read-modify-write so that we can specify which # existing labels need to be deleted. _, bucket_metadata = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Set all old keys' values to None; this will delete each key that # is not included in the new set of labels. merged_labels = dict( (key, None) for key, _ in six.iteritems(label_json)) merged_labels.update(new_label_json) labels_message = LabelTranslation.DictToMessage(merged_labels) else: # ApiSelector.XML # No need to read-modify-write with the XML API. labels_message = LabelTranslation.DictToMessage(new_label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args[1:] for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for bucket_listing_ref in bucket_iter: some_matched = True _SetLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _ChLabel(self): """Parses options and changes labels on the specified buckets.""" self.label_changes = {} self.num_deletions = 0 if self.sub_opts: for o, a in self.sub_opts: if o == '-l': label_split = a.split(':') if len(label_split) != 2: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To add a label, please use the form <key>:<value>.' % a) self.label_changes[label_split[0]] = label_split[1] elif o == '-d': # Ensure only the key is supplied; stop if key:value was given. val_split = a.split(':') if len(val_split) != 1: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To delete a label, provide only its key.' % a) self.label_changes[a] = None self.num_deletions += 1 else: self.RaiseInvalidArgumentException() if not self.label_changes: raise CommandException( 'Please specify at least one label change with the -l or -d flags.') @Retry(PreconditionException, tries=3, timeout_secs=1) def _ChLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # The JSON API's PATCH semantics allow us to skip read-modify-write, # with the exception of one edge case - attempting to delete a # nonexistent label returns an error iff no labels previously existed corrected_changes = self.label_changes if self.num_deletions: (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) if not bucket_metadata.labels: metageneration = bucket_metadata.metageneration # Remove each change that would try to delete a nonexistent key. corrected_changes = dict( (k, v) for k, v in six.iteritems(self.label_changes) if v) labels_message = LabelTranslation.DictToMessage(corrected_changes) else: # ApiSelector.XML # Perform a read-modify-write cycle so that we can specify which # existing labels need to be deleted. (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Modify label_json such that all specified labels are added # (overwriting old labels if necessary) and all specified deletions # are removed from label_json if already present. for key, value in six.iteritems(self.label_changes): if not value and key in label_json: del label_json[key] else: label_json[key] = value labels_message = LabelTranslation.DictToMessage(label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _ChLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetAndPrintLabel(self, bucket_arg): """Gets and prints the labels for a cloud bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( bucket_arg, bucket_fields=['labels']) if bucket_url.scheme == 's3': print((self.gsutil_api.XmlPassThroughGetTagging( bucket_url, provider=bucket_url.scheme))) else: if bucket_metadata.labels: print((LabelTranslation.JsonFromMessage(bucket_metadata.labels, pretty_print=True))) else: print(('%s has no label configuration.' % bucket_url)) def RunCommand(self): """Command entry point for the label command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self._GetAndPrintLabel(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetLabel() elif action_subcommand == 'ch': metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChLabel() else: raise CommandException( 'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".' % (action_subcommand, self.command_name, self.command_name)) return 0 ls.py 0000644 00000065317 15025166704 0005556 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like ls command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import LsHelper from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import text_util from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import LabelTranslation from gslib.utils.unit_util import MakeHumanReadable # Regex that assists with converting JSON timestamp to ls-style output. # This excludes timestamp fractional seconds, for example: # 2013-07-03 20:32:53.048000+00:00 JSON_TIMESTAMP_RE = re.compile(r'([^\s]*)\s([^\.\+]*).*') _SYNOPSIS = """ gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Retrieves a list of providers, buckets, or objects matching the criteria, ordered in the list lexicographically by name. <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> If you run ``gsutil ls`` without URLs, it lists all of the Google Cloud Storage buckets under your default project ID (or all of the Cloud Storage buckets under the project you specify with the ``-p`` flag): gsutil ls If you specify one or more provider URLs, ``gsutil ls`` lists buckets at each listed provider: gsutil ls gs:// gsutil currently supports ``gs://`` and ``s3://`` as valid providers If you specify bucket URLs, or use `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_ to capture a set of buckets, ``gsutil ls`` lists objects at the top level of each bucket, along with the names of each subdirectory. For example: gsutil ls gs://bucket might produce output like: gs://bucket/obj1.htm gs://bucket/obj2.htm gs://bucket/images1/ gs://bucket/images2/ The "/" at the end of the last 2 URLs tells you they are subdirectories, which you can list using: gsutil ls gs://bucket/images* If you specify object URLs, ``gsutil ls`` lists the specified objects. For example: gsutil ls gs://bucket/*.txt lists all files whose name matches the above wildcard at the top level of the bucket. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_. <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> Listing a bucket or subdirectory (as illustrated near the end of the previous section) only shows the objects and names of subdirectories it contains. You can list all objects in a bucket by using the -r option. For example: gsutil ls -r gs://bucket lists the top-level objects and buckets, then the objects and buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. If you want to see all objects in the bucket in one "flat" listing use the recursive ("**") wildcard, like: gsutil ls -r gs://bucket/** or, for a flat listing of a subdirectory: gsutil ls -r gs://bucket/dir/** If you want to see only the subdirectory itself, use the -d option: gsutil ls -d gs://bucket/dir <B>LISTING OBJECT DETAILS</B> If you specify the -l option, gsutil outputs additional information about each matching provider, bucket, subdirectory, or object. For example: gsutil ls -l gs://bucket/*.html gs://bucket/*.txt prints the object size, creation time stamp, and name of each matching object, along with the total count and sum of sizes of all matching objects: 2276224 2020-03-02T19:25:17Z gs://bucket/obj1.html 3914624 2020-03-02T19:30:27Z gs://bucket/obj2.html 131 2020-03-02T19:37:45Z gs://bucket/obj3.txt TOTAL: 3 objects, 6190979 bytes (5.9 MiB) Note that the total listed in parentheses above is in mebibytes (or gibibytes, tebibytes, etc.), which corresponds to the unit of billing measurement for Google Cloud Storage. You can get a listing of all the objects in the top-level bucket directory (along with the total count and sum of sizes) using a command like: gsutil ls -l gs://bucket To print additional detail about objects and buckets use the gsutil ls -L option. For example: gsutil ls -L gs://bucket/obj1 prints something like: gs://bucket/obj1: Creation time: Fri, 26 May 2017 22:55:44 GMT Update time: Tue, 18 Jul 2017 12:31:18 GMT Storage class: STANDARD Content-Length: 60183 Content-Type: image/jpeg Hash (crc32c): zlUhtg== Hash (md5): Bv86IAzFzrD1Z2io/c7yqA== ETag: 5ca67960a586723b7344afffc81 Generation: 1378862725952000 Metageneration: 1 ACL: [ { "entity": "project-owners-867484910061", "projectTeam": { "projectNumber": "867484910061", "team": "owners" }, "role": "OWNER" }, { "email": "jane@gmail.com", "entity": "user-jane@gmail.com", "role": "OWNER" } ] TOTAL: 1 objects, 60183 bytes (58.77 KiB) Note that results may contain additional fields, such as custom metadata or a storage class update time, if they are applicable to the object. Also note that some fields, such as update time, are not available with the (non-default) XML API. See also "gsutil help acl" for getting a more readable version of the ACL. <B>LISTING BUCKET DETAILS</B> If you want to see information about the bucket itself, use the -b option. For example: gsutil ls -L -b gs://bucket prints something like: gs://bucket/ : Storage class: STANDARD Location constraint: US Versioning enabled: False Logging configuration: None Website configuration: None CORS configuration: None Lifecycle configuration: None Requester Pays enabled: True Labels: None Default KMS key: None Time created: Thu, 14 Jan 2016 19:25:17 GMT Time updated: Thu, 08 Jun 2017 21:17:59 GMT Metageneration: 1 Bucket Policy Only enabled: False ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Default ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Note that some fields above (time created, time updated, metageneration) are not available with the (non-default) XML API. <B>OPTIONS</B> -l Prints long listing (owner, length). -L Prints even more detail than -l. Note: If you use this option with the (non-default) XML API it generates an additional request per object being listed, which makes the -L option run much more slowly and cost more than the default JSON API. -d List matching subdirectory names instead of contents, and do not recurse into matching subdirectories even if the -R option is specified. -b Prints info about the bucket when used with a bucket URL. -h When used with -l, prints object sizes in human readable format (e.g., 1 KiB, 234 MiB, 2 GiB, etc.) -p proj_id Specifies the project ID or project number to use for listing buckets. -R, -r Requests a recursive listing, performing at least one listing operation per subdirectory. If you have a large number of subdirectories and do not require recursive-style output ordering, you may be able to instead use wildcards to perform a flat listing, e.g. ``gsutil ls gs://mybucket/**``, which generally performs fewer listing operations. -a Includes non-current object versions / generations in the listing (only useful with a versioning-enabled bucket). If combined with -l option also prints metageneration for each listed object. -e Include ETag in long listing (-l) output. """) class ListingStyle(object): """Enum class for specifying listing style.""" SHORT = 'SHORT' LONG = 'LONG' LONG_LONG = 'LONG_LONG' class LsCommand(Command): """Implementation of gsutil ls command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'ls', command_name_aliases=[ 'dir', 'list', ], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='aebdlLhp:rR', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ls', help_name_aliases=[ 'dir', 'list', ], help_type='command_help', help_one_line_summary='List providers, buckets, or objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # TODO(b/206151616) Add mappings for remaining flags. gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'ls', '--fetch-encrypted-object-hashes'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-l': GcloudStorageFlag('-l'), '-L': GcloudStorageFlag('-L'), '-b': GcloudStorageFlag('-b'), '-e': GcloudStorageFlag('-e'), '-a': GcloudStorageFlag('-a'), '-h': GcloudStorageFlag('--readable-sizes'), '-p': GcloudStorageFlag('--project'), }, ) def _PrintBucketInfo(self, bucket_blr, listing_style): """Print listing info for given bucket. Args: bucket_blr: BucketListingReference for the bucket being listed listing_style: ListingStyle enum describing type of output desired. Returns: Tuple (total objects, total bytes) in the bucket. """ if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): text_util.print_to_fd(bucket_blr) return # listing_style == ListingStyle.LONG_LONG: # We're guaranteed by the caller that the root object is populated. bucket = bucket_blr.root_object location_constraint = bucket.location storage_class = bucket.storageClass fields = { 'bucket': bucket_blr.url_string, 'storage_class': storage_class, 'location_constraint': location_constraint, 'acl': AclTranslation.JsonFromMessage(bucket.acl), 'default_acl': AclTranslation.JsonFromMessage(bucket.defaultObjectAcl), 'versioning': bucket.versioning and bucket.versioning.enabled, 'website_config': 'Present' if bucket.website else 'None', 'logging_config': 'Present' if bucket.logging else 'None', 'cors_config': 'Present' if bucket.cors else 'None', 'lifecycle_config': 'Present' if bucket.lifecycle else 'None', 'requester_pays': bucket.billing and bucket.billing.requesterPays } if bucket.retentionPolicy: fields['retention_policy'] = 'Present' if bucket.labels: fields['labels'] = LabelTranslation.JsonFromMessage(bucket.labels, pretty_print=True) else: fields['labels'] = 'None' if bucket.encryption and bucket.encryption.defaultKmsKeyName: fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName else: fields['default_kms_key'] = 'None' fields['encryption_config'] = 'Present' if bucket.encryption else 'None' # Fields not available in all APIs (e.g. the XML API) if bucket.autoclass and bucket.autoclass.enabled: fields['autoclass_enabled_date'] = ( bucket.autoclass.toggleTime.strftime('%a, %d %b %Y')) if bucket.locationType: fields['location_type'] = bucket.locationType if bucket.customPlacementConfig: fields['custom_placement_locations'] = ( bucket.customPlacementConfig.dataLocations) if bucket.metageneration: fields['metageneration'] = bucket.metageneration if bucket.timeCreated: fields['time_created'] = bucket.timeCreated.strftime( '%a, %d %b %Y %H:%M:%S GMT') if bucket.updated: fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT') if bucket.defaultEventBasedHold: fields['default_eventbased_hold'] = bucket.defaultEventBasedHold if bucket.iamConfiguration: if bucket.iamConfiguration.bucketPolicyOnly: enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled fields['bucket_policy_only_enabled'] = enabled if bucket.iamConfiguration.publicAccessPrevention: fields[ 'public_access_prevention'] = bucket.iamConfiguration.publicAccessPrevention if bucket.rpo: fields['rpo'] = bucket.rpo if bucket.satisfiesPZS: fields['satisfies_pzs'] = bucket.satisfiesPZS # For field values that are multiline, add indenting to make it look # prettier. for key in fields: previous_value = fields[key] if (not isinstance(previous_value, six.string_types) or '\n' not in previous_value): continue new_value = previous_value.replace('\n', '\n\t ') # Start multiline values on a new line if they aren't already. if not new_value.startswith('\n'): new_value = '\n\t ' + new_value fields[key] = new_value # Only display certain properties if the given API returned them (JSON API # returns many fields that the XML API does not). autoclass_line = '' location_type_line = '' custom_placement_locations_line = '' metageneration_line = '' time_created_line = '' time_updated_line = '' default_eventbased_hold_line = '' retention_policy_line = '' bucket_policy_only_enabled_line = '' public_access_prevention_line = '' rpo_line = '' satisifies_pzs_line = '' if 'autoclass_enabled_date' in fields: autoclass_line = '\tAutoclass:\t\t\tEnabled on {autoclass_enabled_date}\n' if 'location_type' in fields: location_type_line = '\tLocation type:\t\t\t{location_type}\n' if 'custom_placement_locations' in fields: custom_placement_locations_line = ( '\tPlacement locations:\t\t{custom_placement_locations}\n') if 'metageneration' in fields: metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n' if 'time_created' in fields: time_created_line = '\tTime created:\t\t\t{time_created}\n' if 'updated' in fields: time_updated_line = '\tTime updated:\t\t\t{updated}\n' if 'default_eventbased_hold' in fields: default_eventbased_hold_line = ( '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n') if 'retention_policy' in fields: retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n' if 'bucket_policy_only_enabled' in fields: bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t' '{bucket_policy_only_enabled}\n') if 'public_access_prevention' in fields: public_access_prevention_line = ('\tPublic access prevention:\t' '{public_access_prevention}\n') if 'rpo' in fields: rpo_line = ('\tRPO:\t\t\t\t{rpo}\n') if 'satisfies_pzs' in fields: satisifies_pzs_line = '\tSatisfies PZS:\t\t\t{satisfies_pzs}\n' text_util.print_to_fd( ('{bucket} :\n' '\tStorage class:\t\t\t{storage_class}\n' + location_type_line + '\tLocation constraint:\t\t{location_constraint}\n' + custom_placement_locations_line + '\tVersioning enabled:\t\t{versioning}\n' '\tLogging configuration:\t\t{logging_config}\n' '\tWebsite configuration:\t\t{website_config}\n' '\tCORS configuration: \t\t{cors_config}\n' '\tLifecycle configuration:\t{lifecycle_config}\n' '\tRequester Pays enabled:\t\t{requester_pays}\n' + retention_policy_line + default_eventbased_hold_line + '\tLabels:\t\t\t\t{labels}\n' + '\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line + time_updated_line + metageneration_line + bucket_policy_only_enabled_line + autoclass_line + public_access_prevention_line + rpo_line + satisifies_pzs_line + '\tACL:\t\t\t\t{acl}\n' '\tDefault ACL:\t\t\t{default_acl}').format(**fields)) if bucket_blr.storage_url.scheme == 's3': text_util.print_to_fd( 'Note: this is an S3 bucket so configuration values may be ' 'blank. To retrieve bucket configuration values, use ' 'individual configuration commands such as gsutil acl get ' '<bucket>.') def _PrintLongListing(self, bucket_listing_ref): """Prints an object with ListingStyle.LONG.""" obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated)) printstr = '%(size)10s %(timestamp)s %(url)s' encoded_etag = None encoded_metagen = None if self.all_versions: printstr += ' metageneration=%(metageneration)s' encoded_metagen = str(obj.metageneration) if self.include_etag: printstr += ' etag=%(etag)s' encoded_etag = obj.etag format_args = { 'size': size_string, 'timestamp': timestamp, 'url': url_str, 'metageneration': encoded_metagen, 'etag': encoded_etag } text_util.print_to_fd(printstr % format_args) return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the ls command.""" got_nomatch_errors = False got_bucket_nomatch_errors = False listing_style = ListingStyle.SHORT get_bucket_info = False self.recursion_requested = False self.all_versions = False self.include_etag = False self.human_readable = False self.list_subdir_contents = True if self.sub_opts: for o, a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-e': self.include_etag = True elif o == '-b': get_bucket_info = True elif o == '-h': self.human_readable = True elif o == '-l': listing_style = ListingStyle.LONG elif o == '-L': listing_style = ListingStyle.LONG_LONG elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-d': self.list_subdir_contents = False if not self.args: # default to listing all gs buckets self.args = ['gs://'] total_objs = 0 total_bytes = 0 def MaybePrintBucketHeader(blr): if len(self.args) > 1: text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string)) print_bucket_header = MaybePrintBucketHeader for url_str in self.args: storage_url = StorageUrlFromString(url_str) if storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_fields = None if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): bucket_fields = ['id'] elif listing_style == ListingStyle.LONG_LONG: bucket_fields = [ 'acl', 'autoclass', 'billing', 'cors', 'customPlacementConfig', 'defaultObjectAcl', 'encryption', 'iamConfiguration', 'labels', 'location', 'locationType', 'logging', 'lifecycle', 'metageneration', 'retentionPolicy', 'defaultEventBasedHold', 'rpo', 'satisfiesPZS', 'storageClass', 'timeCreated', 'updated', 'versioning', 'website', ] if storage_url.IsProvider(): # Provider URL: use bucket wildcard to list buckets. for blr in self.WildcardIterator( '%s://*' % storage_url.scheme).IterBuckets(bucket_fields=bucket_fields): self._PrintBucketInfo(blr, listing_style) elif storage_url.IsBucket() and get_bucket_info: # ls -b bucket listing request: List info about bucket(s). total_buckets = 0 for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): if not ContainsWildcard(url_str) and not blr.root_object: # Iterator does not make an HTTP call for non-wildcarded # listings with fields=='id'. Ensure the bucket exists by calling # GetBucket. self.gsutil_api.GetBucket(blr.storage_url.bucket_name, fields=['id'], provider=storage_url.scheme) self._PrintBucketInfo(blr, listing_style) total_buckets += 1 if not ContainsWildcard(url_str) and not total_buckets: got_bucket_nomatch_errors = True else: # URL names a bucket, object, or object subdir -> # list matching object(s) / subdirs. def _PrintPrefixLong(blr): text_util.print_to_fd('%-33s%s' % ('', six.ensure_text(blr.url_string))) if listing_style == ListingStyle.SHORT: # ls helper by default readies us for a short listing. listing_helper = LsHelper( self.WildcardIterator, self.logger, all_versions=self.all_versions, print_bucket_header_func=print_bucket_header, should_recurse=self.recursion_requested, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG: bucket_listing_fields = [ 'name', 'size', 'timeCreated', 'updated', ] if self.all_versions: bucket_listing_fields.extend([ 'generation', 'metageneration', ]) if self.include_etag: bucket_listing_fields.append('etag') listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=self._PrintLongListing, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG_LONG: # List all fields bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS + ENCRYPTED_FIELDS) listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=PrintFullInfoAboutObject, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) else: raise CommandException('Unknown listing style: %s' % listing_style) exp_dirs, exp_objs, exp_bytes = ( listing_helper.ExpandUrlAndPrint(storage_url)) if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0: got_nomatch_errors = True total_bytes += exp_bytes total_objs += exp_objs if total_objs and listing_style != ListingStyle.SHORT: text_util.print_to_fd( 'TOTAL: %d objects, %d bytes (%s)' % (total_objs, total_bytes, MakeHumanReadable(float(total_bytes)))) if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') if got_bucket_nomatch_errors: raise NotFoundException('One or more bucket URLs matched no buckets.') return 0 defstorageclass.py 0000644 00000017352 15025166704 0010305 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Impl. of default bucket storage class command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil defstorageclass set <storage-class> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defstorageclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "defstorageclass set" command sets the default `storage class <https://cloud.google.com/storage/docs/storage-classes>`_ for the specified bucket(s). If you specify a default storage class for a certain bucket, Google Cloud Storage applies the default storage class to all new objects uploaded to that bucket, except when the storage class is overridden by individual upload requests. Setting a default storage class on a bucket provides a convenient way to ensure newly uploaded objects have a specific storage class. If you don't set the bucket's default storage class, it will default to Standard. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default storage class for a bucket. """ _DESCRIPTION = """ The defstorageclass command has two sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) # The url_string for buckets ends with a slash. # Substitute the last slash with a colon. _GCLOUD_FORMAT_STRING = ('--format=value[separator=\": \"](name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' 'storageClass)') SHIM_GET_COMMAND_MAP = GcloudStorageMap( # Using a list because a string gets splitted up on space and the # format string below has a space. gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ) SHIM_SET_COMMAND_MAP = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--default-storage-class'], flag_map={}, ) class DefStorageClassCommand(Command): """Implementation of gsutil defstorageclass command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defstorageclass', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ # FreeTextArgument allows for using storage class abbreviations. CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defstorageclass', help_name_aliases=['defaultstorageclass'], help_type='command_help', help_one_line_summary='Get or set the default storage class on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': SHIM_GET_COMMAND_MAP, 'set': SHIM_SET_COMMAND_MAP, }, flag_map={}, ) def _CheckIsGsUrl(self, url_str): if not url_str.startswith('gs://'): raise CommandException( '"%s" does not support the URL "%s". Did you mean to use a gs:// ' 'URL?' % (self.command_name, url_str)) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetDefStorageClass(self): """Sets the default storage class for a bucket.""" # At this point, "set" has been popped off the front of self.args. normalized_storage_class = NormalizeStorageClass(self.args[0]) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: some_matched = True bucket_metadata = apitools_messages.Bucket() self.logger.info('Setting default storage class to "%s" for bucket %s' % (normalized_storage_class, blr.url_string.rstrip('/'))) bucket_metadata.storageClass = normalized_storage_class self.gsutil_api.PatchBucket(blr.storage_url.bucket_name, bucket_metadata, provider=blr.storage_url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetDefStorageClass(self): """Gets the default storage class for a bucket.""" # At this point, "get" has been popped off the front of self.args. url_args = self.args some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['storageClass']) for blr in bucket_iter: some_matched = True print('%s: %s' % (blr.url_string.rstrip('/'), blr.root_object.storageClass)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the defstorageclass command.""" action_subcommand = self.args.pop(0) subcommand_args = [action_subcommand] if action_subcommand == 'get': func = self._GetDefStorageClass elif action_subcommand == 'set': func = self._SetDefStorageClass normalized_storage_class = NormalizeStorageClass(self.args[0]) subcommand_args.append(normalized_storage_class) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) metrics.LogCommandParams(subcommands=subcommand_args) func() return 0 rm.py 0000644 00000040001 15025166704 0005535 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rm command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from gslib.cloud_api import BucketNotFoundException from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DecrementFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_PREFIX from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil rm [-f] [-r] url... gsutil rm [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: As part of verifying the existence of objects prior to deletion, ``gsutil rm`` makes ``GET`` requests to Cloud Storage for object metadata. These requests incur `network and operations charges <https://cloud.google.com/storage/pricing>`_. The gsutil rm command removes objects and/or buckets. For example, the following command removes the object ``kitten.png``: gsutil rm gs://bucket/kitten.png Use the -r option to specify recursive object deletion. For example, the following command removes gs://bucket/subdir and all objects and subdirectories under it: gsutil rm -r gs://bucket/subdir When working with versioning-enabled buckets, note that the -r option removes all object versions in the subdirectory. To remove only the live version of each object in the subdirectory, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_. The following command removes all versions of all objects in a bucket, and then deletes the bucket: gsutil rm -r gs://bucket To remove all objects and their versions from a bucket without deleting the bucket, use the ``-a`` option: gsutil rm -a gs://bucket/** If you have a large number of objects to remove, use the ``gsutil -m`` option, which enables multi-threading/multi-processing: gsutil -m rm -r gs://my_bucket/subdir You can pass a list of URLs (one per line) to remove on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to remove objects identified by a program, such as: some_program | gsutil -m rm -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. Note that ``gsutil rm`` refuses to remove files from the local file system. For example, this fails: gsutil rm *.txt WARNING: Object removal cannot be undone. Cloud Storage is designed to give developers a high amount of flexibility and control over their data, and Google maintains strict controls over the processing and purging of deleted data. If you have concerns that your application software or your users may at some point erroneously delete or replace data, see `Options for controlling data lifecycles <https://cloud.google.com/storage/docs/control-data-lifecycles>`_ for ways to protect your data from accidental data deletion. <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing multiple objects. If some of the objects could not be removed, gsutil's exit status will be non-zero even if this flag is set. Execution will still halt if an inaccessible bucket is encountered. This option is implicitly set when running "gsutil -m rm ...". -I Causes gsutil to read the list of objects to remove from stdin. This allows you to run a program that generates the list of objects to remove. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents (all objects and subdirectories that it contains) to be removed recursively. If used with a bucket-only URL (like gs://bucket), after deleting objects and subdirectories gsutil deletes the bucket. This option implies the -a option and deletes all object versions. If you only want to delete live object versions, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_ instead of -r. -a Delete all versions of an object. """) def _RemoveExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) # TODO: Use shared state to track missing bucket names when we get a # BucketNotFoundException. Then improve bucket removal logic and exception # messages. if isinstance(e, BucketNotFoundException): cls.bucket_not_found_count += 1 cls.logger.error(str(e)) else: if _ExceptionMatchesBucketToDelete(cls.bucket_strings_to_delete, e): DecrementFailureCount() else: cls.op_failure_count += 1 # pylint: disable=unused-argument def _RemoveFoldersExceptionHandler(cls, e): """When removing folders, we don't mind if none exist.""" if ((isinstance(e, CommandException) and NO_URLS_MATCHED_PREFIX in e.reason) or isinstance(e, NotFoundException)): DecrementFailureCount() else: raise e def _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RemoveFunc(name_expansion_result, thread_state=thread_state) def _ExceptionMatchesBucketToDelete(bucket_strings_to_delete, e): """Returns True if the exception matches a bucket slated for deletion. A recursive delete call on an empty bucket will raise an exception when listing its objects, but if we plan to delete the bucket that shouldn't result in a user-visible error. Args: bucket_strings_to_delete: Buckets slated for recursive deletion. e: Exception to check. Returns: True if the exception was a no-URLs-matched exception and it matched one of bucket_strings_to_delete, None otherwise. """ if bucket_strings_to_delete: msg = NO_URLS_MATCHED_TARGET % '' if msg in str(e): parts = str(e).split(msg) return len(parts) == 2 and parts[1] in bucket_strings_to_delete class RmCommand(Command): """Implementation of gsutil rm command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rm', command_name_aliases=['del', 'delete', 'remove'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=constants.NO_MAX, supported_sub_args='afIrR', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rm', help_name_aliases=['del', 'delete', 'remove'], help_type='command_help', help_one_line_summary='Remove objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'rm'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-a': GcloudStorageFlag('-a'), '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rm command.""" # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). self.continue_on_error = self.parallel_operations self.read_args_from_stdin = False self.all_versions = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-I': self.read_args_from_stdin = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rm command (without -I) expects at ' 'least one URL.') url_strs = self.args # Tracks number of object deletes that failed. self.op_failure_count = 0 # Tracks if any buckets were missing. self.bucket_not_found_count = 0 # Tracks buckets that are slated for recursive deletion. bucket_urls_to_delete = [] self.bucket_strings_to_delete = [] if self.recursion_requested: bucket_fields = ['id'] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsBucket() or url.IsProvider(): for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): bucket_urls_to_delete.append(blr.storage_url) self.bucket_strings_to_delete.append(url_str) self.preconditions = PreconditionsFromHeaders(self.headers or {}) try: # Expand wildcards, dirs, buckets, and bucket subdirs in URLs. name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Perform remove requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count', 'bucket_not_found_count'], seek_ahead_iterator=seek_ahead_iterator) # Assuming the bucket has versioning enabled, url's that don't map to # objects should throw an error even with all_versions, since the prior # round of deletes only sends objects to a history table. # This assumption that rm -a is only called for versioned buckets should be # corrected, but the fix is non-trivial. except CommandException as e: # Don't raise if there are buckets to delete -- it's valid to say: # gsutil rm -r gs://some_bucket # if the bucket is empty. if _ExceptionMatchesBucketToDelete(self.bucket_strings_to_delete, e): DecrementFailureCount() else: raise except ServiceException as e: if not self.continue_on_error: raise if self.bucket_not_found_count: raise CommandException('Encountered non-existent bucket during listing') if self.op_failure_count and not self.continue_on_error: raise CommandException('Some files could not be removed.') # If this was a gsutil rm -r command covering any bucket subdirs, # remove any dir_$folder$ objects (which are created by various web UI # tools to simulate folders). if self.recursion_requested: folder_object_wildcards = [] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsObject(): folder_object_wildcards.append(url_str.rstrip('*') + '*_$folder$') if folder_object_wildcards: self.continue_on_error = True try: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, folder_object_wildcards, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions) # When we're removing folder objects, always continue on error self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveFoldersExceptionHandler, fail_on_error=False) except CommandException as e: # Ignore exception from name expansion due to an absent folder file. if not e.reason.startswith(NO_URLS_MATCHED_PREFIX): raise # Now that all data has been deleted, delete any bucket URLs. for url in bucket_urls_to_delete: self.logger.info('Removing %s...', url) @Retry(NotEmptyException, tries=3, timeout_secs=1) def BucketDeleteWithRetry(): self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) BucketDeleteWithRetry() if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be removed.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RemoveFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Removing %s...', exp_src_url) try: gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, preconditions=self.preconditions, generation=exp_src_url.generation, provider=exp_src_url.scheme) except NotFoundException as e: # DeleteObject will sometimes return a 504 (DEADLINE_EXCEEDED) when # the operation was in fact successful. When a retry is attempted in # these cases, it will fail with a (harmless) 404. The 404 is harmless # since it really just means the file was already deleted, which is # what we want anyway. Here we simply downgrade the message to info # rather than error and correct the command-level failure total. self.logger.info('Cannot find %s', exp_src_url) DecrementFailureCount() _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) logging.py 0000644 00000031674 15025166704 0006565 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of logging configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import text_util _SET_SYNOPSIS = """ gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... gsutil logging set off gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil logging get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n' _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command has two sub-commands: <B>ON</B> The ``gsutil logging set on`` command enables usage and storage logging for the buckets named by the specified URLs, outputting log files to the bucket specified with the ``-b`` flag. Cloud Storage doesn't validate the existence of the output bucket, so users should ensure it already exists, and all URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). The optional ``-o`` flag specifies the prefix for log object names. The default prefix is the bucket name. For example, the command: gsutil logging set on -b gs://my_logging_bucket -o UsageLog \\ gs://my_bucket1 gs://my_bucket2 causes all read and write activity to objects in ``gs://mybucket1`` and ``gs://mybucket2`` to be logged to objects prefixed with the name ``UsageLog``, with those log objects written to the bucket ``gs://my_logging_bucket``. In addition to enabling logging on your bucket(s), you also need to grant cloud-storage-analytics@google.com write access to the log bucket, using this command: gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket Note that log data may contain sensitive information, so you should make sure to set an appropriate default bucket ACL to protect that data. (See "gsutil help defacl".) <B>OFF</B> This command disables usage and storage logging for the buckets named by the specified URLs. All URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). No logging data is removed from the log buckets when you disable logging, but Google Cloud Storage stops delivering new logs once you have run this command. """ _GET_DESCRIPTION = """ <B>GET</B> If logging is enabled for the specified bucket url, the server responds with a JSON document that looks something like this: { "logBucket": "my_logging_bucket", "logObjectPrefix": "UsageLog" } You can download log data from your log bucket using the gsutil cp command. """ _DESCRIPTION = """ Google Cloud Storage offers `usage logs and storage logs <https://cloud.google.com/storage/docs/access-logs>`_ in the form of CSV files that you can download and view. Usage logs provide information for all of the requests made on a specified bucket and are created hourly. Storage logs provide information about the storage consumption of that bucket for the last day and are created daily. Once set up, usage logs and storage logs are automatically created as new objects in a bucket that you specify. Usage logs and storage logs are subject to the same pricing as other objects stored in Cloud Storage. For a complete list of usage log fields and storage data fields, see `Usage and storage log format <https://cloud.google.com/storage/docs/access-logs#format>`_. The ``logging`` command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION + """ <B>OPTIONS</B> -b bucket_name Specifies the bucket that stores the generated logs. This flag is only available for the ``set on`` command and is required for that command. -o log_prefix Specifies a common prefix for the names of generated logs. This flag is only available for the ``set on`` command and is optional for that command. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LoggingCommand(Command): """Implementation of gsutil logging command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'logging', command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='b:o:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='logging', help_name_aliases=[ 'loggingconfig', 'logs', 'log', 'getlogging', 'enablelogging', 'disablelogging', ], help_type='command_help', help_one_line_summary='Configure or retrieve logging on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=logging_config,empty=\' has ' 'no logging configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-b': GcloudStorageFlag('--log-bucket'), '-o': GcloudStorageFlag('--log-object-prefix'), }, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-log-bucket', '--clear-log-object-prefix', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _Get(self): """Gets logging configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['logging']) if bucket_url.scheme == 's3': text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging( bucket_url, provider=bucket_url.scheme), end='') else: if (bucket_metadata.logging and bucket_metadata.logging.logBucket and bucket_metadata.logging.logObjectPrefix): text_util.print_to_fd( str(encoding.MessageToJson(bucket_metadata.logging))) else: text_util.print_to_fd('%s has no logging configuration.' % bucket_url) return 0 def _Enable(self): """Enables logging configuration for a bucket.""" # Disallow multi-provider 'logging set on' calls, because the schemas # differ. if not UrlsAreForSingleProvider(self.args): raise CommandException('"logging set on" command spanning providers not ' 'allowed.') target_bucket_url = None target_prefix = None for opt, opt_arg in self.sub_opts: if opt == '-b': target_bucket_url = StorageUrlFromString(opt_arg) if opt == '-o': target_prefix = opt_arg if not target_bucket_url: raise CommandException('"logging set on" requires \'-b <log_bucket>\' ' 'option') if not target_bucket_url.IsBucket(): raise CommandException('-b option must specify a bucket URL.') # Iterate over URLs, expanding wildcards and setting logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Enabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue( logBucket=target_bucket_url.bucket_name, logObjectPrefix=target_prefix or url.bucket_name) bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def _Disable(self): """Disables logging configuration for a bucket.""" # Iterate over URLs, expanding wildcards, and disabling logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Disabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue() bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def RunCommand(self): """Command entry point for the logging command.""" # Parse the subcommand and alias for the new logging command. action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._Get metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': state_subcommand = self.args.pop(0) if not self.args: self.RaiseWrongNumberOfArgumentsException() if state_subcommand == 'on': func = self._Enable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) elif state_subcommand == 'off': func = self._Disable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) else: raise CommandException( ('Invalid subcommand "%s" for the "%s %s" command.\n' 'See "gsutil help logging".') % (state_subcommand, self.command_name, action_subcommand)) else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help logging".') % (action_subcommand, self.command_name)) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) func() return 0 compose.py 0000644 00000014442 15025166704 0006576 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of compose command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto import config from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_COMPOSE_ARITY = 32 _SYNOPSIS = """ gsutil compose gs://bucket/source_obj1 [gs://bucket/source_obj2 ...] gs://bucket/composite_obj """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The compose command creates a new object whose content is the concatenation of a given sequence of source objects under the same bucket. gsutil uses the content type of the first source object to determine the destination object's content type and does not modify or delete the source objects as part of the compose command. For more information, see the `composite objects topic <https://cloud.google.com/storage/docs/composite-objects>`_. There is a limit (currently %d) to the number of components that can be composed in a single operation. """ % (MAX_COMPOSE_ARITY)) class ComposeCommand(Command): """Implementation of gsutil compose command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'compose', command_name_aliases=['concat'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=MAX_COMPOSE_ARITY + 1, supported_sub_args='', # Not files, just object names without gs:// prefix. file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='compose', help_name_aliases=['concat'], help_type='command_help', help_one_line_summary=( 'Concatenate a sequence of objects into a new composite object.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'compose'], flag_map={}, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"compose" called on URL with unsupported provider (%s).' % str(url)) # Command entry point. def RunCommand(self): """Command entry point for the compose command.""" target_url_str = self.args[-1] self.args = self.args[:-1] target_url = StorageUrlFromString(target_url_str) self.CheckProvider(target_url) if target_url.HasGeneration(): raise CommandException('A version-specific URL (%s) cannot be ' 'the destination for gsutil compose - abort.' % target_url) dst_obj_metadata = apitools_messages.Object(name=target_url.object_name, bucket=target_url.bucket_name) components = [] # Remember the first source object so we can get its content type. first_src_url = None for src_url_str in self.args: if ContainsWildcard(src_url_str): src_url_iter = self.WildcardIterator(src_url_str).IterObjects() else: src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))] for blr in src_url_iter: src_url = blr.storage_url self.CheckProvider(src_url) if src_url.bucket_name != target_url.bucket_name: raise CommandException('GCS does not support inter-bucket composing.') if not first_src_url: first_src_url = src_url src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=src_url.object_name)) if src_url.HasGeneration(): src_obj_metadata.generation = int(src_url.generation) components.append(src_obj_metadata) # Avoid expanding too many components, and sanity check each name # expansion result. if len(components) > MAX_COMPOSE_ARITY: raise CommandException('"compose" called with too many component ' 'objects. Limit is %d.' % MAX_COMPOSE_ARITY) if not components: raise CommandException('"compose" requires at least 1 component object.') first_src_obj_metadata = self.gsutil_api.GetObjectMetadata( first_src_url.bucket_name, first_src_url.object_name, provider=first_src_url.scheme, fields=['contentEncoding', 'contentType']) dst_obj_metadata.contentType = first_src_obj_metadata.contentType dst_obj_metadata.contentEncoding = first_src_obj_metadata.contentEncoding preconditions = PreconditionsFromHeaders(self.headers or {}) self.logger.info('Composing %s from %d component object(s).', target_url, len(components)) self.gsutil_api.ComposeObject( components, dst_obj_metadata, preconditions=preconditions, provider=target_url.scheme, encryption_tuple=GetEncryptionKeyWrapper(config)) signurl.py 0000644 00000071613 15025166704 0006617 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of URL Signing workflow. see: https://cloud.google.com/storage/docs/access-control#Signed-URLs) """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar import copy from datetime import datetime from datetime import timedelta from datetime import timezone import getpass import json import re import sys import six from six.moves import urllib from apitools.base.py.exceptions import HttpError from apitools.base.py.http_wrapper import MakeRequest from apitools.base.py.http_wrapper import Request from boto import config from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import constants from gslib.utils.boto_util import GetNewHttp from gslib.utils.shim_util import GcloudStorageMap, GcloudStorageFlag from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl, to_bytes try: # Check for openssl. # pylint: disable=C6204 from OpenSSL.crypto import FILETYPE_PEM from OpenSSL.crypto import load_privatekey from OpenSSL.crypto import sign from OpenSSL.crypto import PKey HAVE_OPENSSL = True except ImportError: load_privatekey = None sign = None HAVE_OPENSSL = False FILETYPE_PEM = None try: from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import pkcs12 from cryptography.hazmat._oid import NameOID HAVE_CRYPTO = True except ImportError: HAVE_CRYPTO = False _CRYPTO_IMPORT_ERROR = "pyca/cryptography is not available. Either install it, or please consider using the .json keyfile" _AUTO_DETECT_REGION = 'auto' _MAX_EXPIRATION_TIME = timedelta(days=7) _MAX_EXPIRATION_TIME_WITH_MINUS_U = timedelta(hours=12) _SYNOPSIS = """ gsutil signurl [-c <content_type>] [-d <duration>] [-m <http_method>] \\ [-p <password>] [-r <region>] [-b <project>] (-u | <private-key-file>) \\ (gs://<bucket_name> | gs://<bucket_name>/<object_name>)... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The signurl command will generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. Please see the `Signed URLs documentation <https://cloud.google.com/storage/docs/access-control/signed-urls>`_ for background about signed URLs. Multiple gs:// URLs may be provided and may contain wildcards. A signed URL will be produced for each provided URL, authorized for the specified HTTP method and valid for the given duration. NOTE: Unlike the gsutil ls command, the signurl command does not support operations on sub-directories. For example, unless you have an object named ``some-directory/`` stored inside the bucket ``some-bucket``, the following command returns an error: ``gsutil signurl <private-key-file> gs://some-bucket/some-directory/`` The signurl command uses the private key for a service account (the '<private-key-file>' argument) to generate the cryptographic signature for the generated URL. The private key file must be in PKCS12 or JSON format. If the private key is encrypted the signed URL command will prompt for the passphrase used to protect the private key file (default 'notasecret'). For more information regarding generating a private key for use with the signurl command please see the `Authentication documentation. <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_ If you used `service account credentials <https://cloud.google.com/storage/docs/gsutil/addlhelp/CredentialTypesSupportingVariousUseCases#supported-credential-types_1>`_ for authentication, you can replace the <private-key-file> argument with the -u or --use-service-account option to use the system-managed private key directly. This avoids the need to store a private key file locally, but prior to using this flag you must `configure <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ ``gcloud`` to use your service account credentials. <B>OPTIONS</B> -b <project> Allows you to specify a user project that will be billed for requests that use the signed URL. This is useful for generating presigned links for buckets that use requester pays. Note that it's not valid to specify both the ``-b`` and ``--use-service-account`` options together. -c Specifies the content type for which the signed URL is valid for. -d Specifies the duration that the signed URL should be valid for, default duration is 1 hour. Times may be specified with no suffix (default hours), or with s = seconds, m = minutes, h = hours, d = days. This option may be specified multiple times, in which case the duration the link remains valid is the sum of all the duration options. The max duration allowed is 7 days when ``private-key-file`` is used. The max duration allowed is 12 hours when -u option is used. This limitation exists because the system-managed key used to sign the URL may not remain valid after 12 hours. -m Specifies the HTTP method to be authorized for use with the signed URL, default is GET. You may also specify RESUMABLE to create a signed resumable upload start URL. When using a signed URL to start a resumable upload session, you will need to specify the 'x-goog-resumable:start' header in the request or else signature validation will fail. -p Specify the private key password instead of prompting. -r <region> Specifies the `region <https://cloud.google.com/storage/docs/locations>`_ in which the resources for which you are creating signed URLs are stored. Default value is 'auto' which will cause gsutil to fetch the region for the resource. When auto-detecting the region, the current gsutil user's credentials, not the credentials from the private-key-file, are used to fetch the bucket's metadata. This option must be specified and not 'auto' when generating a signed URL to create a bucket. -u Use service account credentials instead of a private key file to sign the URL. You can also use the ``--use-service-account`` option, which is equivalent to ``-u``. Note that both options have a maximum allowed duration of 12 hours for a valid link. <B>USAGE</B> Create a signed URL for downloading an object valid for 10 minutes: gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object> Create a signed URL, valid for one hour, for uploading a plain text file via HTTP PUT: gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to PUT to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to POST a resumable upload to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m RESUMABLE -d 1d -c image/jpg <private-key-file> \\ gs://bucket/<obj> """) def _NowUTC(): """Returns the current utc time as a datetime object.""" return datetime.now(tz=timezone.utc).replace(tzinfo=None) def _DurationToTimeDelta(duration): r"""Parses the given duration and returns an equivalent timedelta.""" match = re.match(r'^(\d+)([dDhHmMsS])?$', duration) if not match: raise CommandException('Unable to parse duration string') duration, modifier = match.groups('h') duration = int(duration) modifier = modifier.lower() if modifier == 'd': ret = timedelta(days=duration) elif modifier == 'h': ret = timedelta(hours=duration) elif modifier == 'm': ret = timedelta(minutes=duration) elif modifier == 's': ret = timedelta(seconds=duration) return ret def _GenSignedUrl(key, api, use_service_account, provider, client_id, method, duration, gcs_path, logger, region, content_type=None, billing_project=None, string_to_sign_debug=False, generation=None): """Construct a string to sign with the provided key. Args: key: The private key to use for signing the URL. api: The CloudApiDelegator instance use_service_account: If True, use the service account credentials instead of using the key file to sign the URL provider: Cloud storage provider to connect to. If not present, class-wide default is used. client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. gcs_path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. content_type: Optional Content-Type for the signed URL. HTTP requests using the URL must match this Content-Type. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. generation: If not None, specifies a version of an object for signing. Returns: The complete URL (string). """ gs_host = config.get('Credentials', 'gs_host', 'storage.googleapis.com') signed_headers = {'host': gs_host} if method == 'RESUMABLE': method = 'POST' signed_headers['x-goog-resumable'] = 'start' if not content_type: logger.warn('Warning: no Content-Type header was specified with the -c ' 'flag, so uploads to the resulting Signed URL must not ' 'specify a Content-Type.') if content_type: signed_headers['content-type'] = content_type if use_service_account: final_url = api.SignUrl(provider=provider, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) else: if six.PY2: digest = b'RSA-SHA256' else: # Your IDE may complain about this due to a bad docstring in pyOpenSsl: # https://github.com/pyca/pyopenssl/issues/741 digest = 'RSA-SHA256' string_to_sign, canonical_query_string = CreatePayload( client_id=client_id, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, billing_project=billing_project, string_to_sign_debug=string_to_sign_debug) if isinstance(key, PKey): raw_signature = sign(key, string_to_sign, digest) else: if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) raw_signature = key.sign(to_bytes(string_to_sign), padding.PKCS1v15(), hashes.SHA256()) final_url = GetFinalUrl(raw_signature, gs_host, gcs_path, canonical_query_string) return final_url def _ReadKeystore(key_string, passwd): key_string = to_bytes(key_string) passwd = to_bytes(passwd) # Cryptography package is not bundled with gsutil, Keeping it at top would throw error. if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) try: key, cert, add = pkcs12.load_key_and_certificates(key_string, passwd) client_email = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value return key, client_email except: raise CommandException('Unable to load the keyfile, Invalid password or PKCS12 data.') def _ReadJSONKeystore(ks_contents, passwd=None): """Read the client email and private key from a JSON keystore. Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console aren't encrypted so the passwd is optional as load_privatekey will prompt for the PEM passphrase if the key is encrypted. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must be a valid JSON string and contain the fields 'private_key' and 'client_email'. passwd: Passphrase for encrypted private keys. Returns: key: Parsed private key from the keystore. client_email: The email address for the service account. Raises: ValueError: If unable to parse ks_contents or keystore is missing required fields. """ # ensuring that json.loads receives unicode in Python 3 and bytes in Python 2 # Previous to Python 3.6, there was no automatic conversion and str was req. ks = json.loads(six.ensure_str(ks_contents)) if 'client_email' not in ks or 'private_key' not in ks: raise ValueError('JSON keystore doesn\'t contain required fields') client_email = ks['client_email'] if passwd: key = load_privatekey(FILETYPE_PEM, ks['private_key'], passwd) else: key = load_privatekey(FILETYPE_PEM, ks['private_key']) return key, client_email class UrlSignCommand(Command): """Implementation of gsutil url_sign command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'signurl', command_name_aliases=['signedurl', 'queryauth'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='m:d:b:c:p:r:u', supported_private_args=['use-service-account'], file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreFileURLsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='signurl', help_name_aliases=[ 'signedurl', 'queryauth', ], help_type='command_help', help_one_line_summary='Create a signed URL', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): # TODO (b/287273841): Replace `copy` with a better pattern for # translating flag values. original_args = copy.deepcopy(self.args) original_sub_opts = copy.deepcopy(self.sub_opts) gcloud_command = [ 'storage', 'sign-url', '--format=csv[separator="\\t"](resource:label="URL",' ' http_verb:label="HTTP Method",' ' expiration:label="Expiration",' ' signed_url:label="Signed URL")', '--private-key-file=' + self.args[0] ] self.args = self.args[1:] duration_arg_idx = None http_verb_arg_idx = None content_type_arg_idx = None billing_project_arg_idx = None for i, (flag, _) in enumerate(self.sub_opts): if flag == '-d': duration_arg_idx = i elif flag == '-m': http_verb_arg_idx = i elif flag == '-c': content_type_arg_idx = i elif flag == '-b': billing_project_arg_idx = i if duration_arg_idx is not None: # Convert duration to seconds, which gcloud can handle. seconds = str( int( _DurationToTimeDelta( self.sub_opts[duration_arg_idx][1]).total_seconds())) + 's' self.sub_opts[duration_arg_idx] = ('-d', seconds) if http_verb_arg_idx is not None: if self.sub_opts[http_verb_arg_idx][1] == 'RESUMABLE': self.sub_opts[http_verb_arg_idx] = ('-m', 'POST') gcloud_command += ['--headers=x-goog-resumable=start'] if content_type_arg_idx is not None: content_type_value = self.sub_opts[content_type_arg_idx][1] self.sub_opts[content_type_arg_idx] = ('-c', 'content-type=' + content_type_value) if billing_project_arg_idx is not None: project_value = self.sub_opts[billing_project_arg_idx][1] self.sub_opts[billing_project_arg_idx] = ('-b', 'userProject=' + project_value) fully_translated_command = super().get_gcloud_storage_args( GcloudStorageMap(gcloud_command=gcloud_command, flag_map={ '-m': GcloudStorageFlag('--http-verb'), '-d': GcloudStorageFlag('--duration'), '-b': GcloudStorageFlag('--query-params'), '-c': GcloudStorageFlag('--headers'), '-r': GcloudStorageFlag('--region'), '-p': GcloudStorageFlag('--private-key-password'), })) # Ensures dry run mode works correctly, as flag translation requires # mutating command state. # TODO(b/287273841): Refactor so that there's a better pattern for # translating flag values. self.args = original_args self.sub_opts = original_sub_opts return fully_translated_command def _ParseAndCheckSubOpts(self): # Default argument values delta = None method = 'GET' content_type = '' passwd = None region = _AUTO_DETECT_REGION use_service_account = False billing_project = None for o, v in self.sub_opts: # TODO(PY3-ONLY): Delete this if block. if six.PY2: v = v.decode(sys.stdin.encoding or constants.UTF8) if o == '-d': if delta is not None: delta += _DurationToTimeDelta(v) else: delta = _DurationToTimeDelta(v) elif o == '-m': method = v elif o == '-c': content_type = v elif o == '-p': passwd = v elif o == '-r': region = v elif o == '-u' or o == '--use-service-account': use_service_account = True elif o == '-b': billing_project = v else: self.RaiseInvalidArgumentException() if delta is None: delta = timedelta(hours=1) else: if use_service_account and delta > _MAX_EXPIRATION_TIME_WITH_MINUS_U: # This restriction comes from the IAM SignBlob API. The SignBlob # API uses a system-managed key which can guarantee validation only # up to 12 hours. b/156160482#comment4 raise CommandException( 'Max valid duration allowed is %s when -u flag is used. For longer' ' duration, consider using the private-key-file instead of the -u' ' option.' % _MAX_EXPIRATION_TIME_WITH_MINUS_U) elif delta > _MAX_EXPIRATION_TIME: raise CommandException('Max valid duration allowed is ' '%s' % _MAX_EXPIRATION_TIME) if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']: raise CommandException('HTTP method must be one of' '[GET|HEAD|PUT|DELETE|RESUMABLE]') if not use_service_account and len(self.args) < 2: raise CommandException( 'The command requires a key file argument and one or more ' 'URL arguments if the --use-service-account flag is missing. ' 'Run `gsutil help signurl` for more info') if use_service_account and billing_project: raise CommandException( 'Specifying both the -b and --use-service-account options together is' 'invalid.') return method, delta, content_type, passwd, region, use_service_account, billing_project def _ProbeObjectAccessWithClient(self, key, use_service_account, provider, client_email, gcs_path, generation, logger, region, billing_project): """Performs a head request against a signed URL to check for read access.""" # Choose a reasonable time in the future; if the user's system clock is # 60 or more seconds behind the server's this will generate an error. signed_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=provider, client_id=client_email, method='HEAD', duration=timedelta(seconds=60), gcs_path=gcs_path, generation=generation, logger=logger, region=region, billing_project=billing_project, string_to_sign_debug=True) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError.FromResponse(response) return response.status_code except HttpError as http_error: if http_error.has_attr('response'): error_response = http_error.response error_string = ('Unexpected HTTP response code %s while querying ' 'object readability. Is your system clock accurate?' % error_response.status_code) if error_response.content: error_string += ' Content: %s' % error_response.content else: error_string = ('Expected an HTTP response code of ' '200 while querying object readability, but received ' 'an error: %s' % http_error) raise CommandException(error_string) def _EnumerateStorageUrls(self, in_urls): ret = [] for url_str in in_urls: if ContainsWildcard(url_str): ret.extend([blr.storage_url for blr in self.WildcardIterator(url_str)]) else: ret.append(StorageUrlFromString(url_str)) return ret def RunCommand(self): """Command entry point for signurl command.""" if not HAVE_OPENSSL: raise CommandException( 'The signurl command requires the pyopenssl library (try pip ' 'install pyopenssl or easy_install pyopenssl)') method, delta, content_type, passwd, region, use_service_account, billing_project = ( self._ParseAndCheckSubOpts()) arg_start_index = 0 if use_service_account else 1 storage_urls = self._EnumerateStorageUrls(self.args[arg_start_index:]) region_cache = {} key = None if not use_service_account: try: key, client_email = _ReadJSONKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: # Ignore and try parsing as a pkcs12. if not passwd: passwd = getpass.getpass('Keystore password:') try: key, client_email = _ReadKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: raise CommandException('Unable to parse private key from {0}'.format( self.args[0])) else: client_email = self.gsutil_api.GetServiceAccountId(provider='gs') print('URL\tHTTP Method\tExpiration\tSigned URL') for url in storage_urls: if url.scheme != 'gs': raise CommandException('Can only create signed urls from gs:// urls') if url.IsBucket(): if region == _AUTO_DETECT_REGION: raise CommandException('Generating signed URLs for creating buckets' ' requires a region be specified via the -r ' 'option. Run `gsutil help signurl` for more ' 'information about the \'-r\' option.') gcs_path = url.bucket_name if method == 'RESUMABLE': raise CommandException('Resumable signed URLs require an object ' 'name.') else: # Need to URL encode the object name as Google Cloud Storage does when # computing the string to sign when checking the signature. gcs_path = '{0}/{1}'.format( url.bucket_name, urllib.parse.quote(url.object_name.encode(constants.UTF8), safe=b'/~')) if region == _AUTO_DETECT_REGION: if url.bucket_name in region_cache: bucket_region = region_cache[url.bucket_name] else: try: _, bucket = self.GetSingleBucketUrlFromArg( 'gs://{}'.format(url.bucket_name), bucket_fields=['location']) except Exception as e: raise CommandException( '{}: Failed to auto-detect location for bucket \'{}\'. Please ' 'ensure you have storage.buckets.get permission on the bucket ' 'or specify the bucket\'s location using the \'-r\' option.'. format(e.__class__.__name__, url.bucket_name)) bucket_region = bucket.location.lower() region_cache[url.bucket_name] = bucket_region else: bucket_region = region final_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=url.scheme, client_id=client_email, method=method, duration=delta, gcs_path=gcs_path, generation=url.generation, logger=self.logger, region=bucket_region, content_type=content_type, billing_project=billing_project, string_to_sign_debug=True) expiration = calendar.timegm((datetime.now(tz=timezone.utc).replace(tzinfo=None) + delta).utctimetuple()) expiration_dt = datetime.fromtimestamp(expiration) time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S') # TODO(PY3-ONLY): Delete this if block. if six.PY2: time_str = time_str.decode(constants.UTF8) url_info_str = '{0}\t{1}\t{2}\t{3}'.format(url.url_string, method, time_str, final_url) # TODO(PY3-ONLY): Delete this if block. if six.PY2: url_info_str = url_info_str.encode(constants.UTF8) print(url_info_str) response_code = self._ProbeObjectAccessWithClient( key, use_service_account, url.scheme, client_email, gcs_path, url.generation, self.logger, bucket_region, billing_project) if response_code == 404: if url.IsBucket() and method != 'PUT': raise CommandException( 'Bucket {0} does not exist. Please create a bucket with ' 'that name before a creating signed URL to access it.'.format( url)) else: if method != 'PUT' and method != 'RESUMABLE': raise CommandException( 'Object {0} does not exist. Please create/upload an object ' 'with that name before a creating signed URL to access it.'. format(url)) elif response_code == 403: self.logger.warn( '%s does not have permissions on %s, using this link will likely ' 'result in a 403 error until at least READ permissions are granted', client_email or 'The account', url) return 0 rsync.py 0000644 00000237010 15025166704 0006265 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rsync command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import errno import heapq import io from itertools import islice import logging import os import re import tempfile import textwrap import time import traceback import sys import six from six.moves import urllib from boto import config import crcmod from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import RegisterSignalHandler from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import constants from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.hashing_helper import CalculateB64EncodedCrc32cFromContents from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import SLOW_CRCMOD_RSYNC_WARNING from gslib.utils.hashing_helper import SLOW_CRCMOD_WARNING from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import NA_ID from gslib.utils.posix_util import NA_MODE from gslib.utils.posix_util import NA_TIME from gslib.utils.posix_util import NeedsPOSIXAttributeUpdate from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import UID_ATTR from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.posix_util import WarnFutureTimestamp from gslib.utils.posix_util import WarnInvalidValue from gslib.utils.posix_util import WarnNegativeAttribute from gslib.utils.rsync_util import DiffAction from gslib.utils.rsync_util import RsyncDiffToApply from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import CopyCustomMetadata from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.unit_util import TEN_MIB from gslib.wildcard_iterator import CreateWildcardIterator if six.PY3: long = int _SYNOPSIS = """ gsutil rsync [OPTION]... src_url dst_url """ # pylint: disable=anomalous-backslash-in-string _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rsync command makes the contents under dst_url the same as the contents under src_url, by copying any missing files/objects (or those whose data has changed), and (if the -d option is specified) deleting any extra files/objects. src_url must specify a directory, bucket, or bucket subdirectory. For example, to sync the contents of the local directory "data" to the bucket gs://mybucket/data, you could do: gsutil rsync data gs://mybucket/data To recurse into directories use the -r option: gsutil rsync -r data gs://mybucket/data If you have a large number of objects to synchronize you might want to use the gsutil -m option (see "gsutil help options"), to perform parallel (multi-threaded/multi-processing) synchronization: gsutil -m rsync -r data gs://mybucket/data The -m option typically will provide a large performance boost if either the source or destination (or both) is a cloud URL. If both source and destination are file URLs the -m option will typically thrash the disk and slow synchronization down. Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways that can be surprising. Also, attempting to copy files whose names contain wildcard characters can result in problems. For more details about these issues see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. Note 2: If you are synchronizing a large amount of data between clouds you might consider setting up a `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_ account and running gsutil there. Since cross-provider gsutil data transfers flow through the machine where gsutil is running, doing this can make your transfer run significantly faster than running gsutil on your local workstation. Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a `flat namespace <https://cloud.google.com/storage/docs/folders>`_. <B>Using -d Option (with caution!) to mirror source and destination.</B> The rsync -d option is very useful and commonly used, because it provides a means of making the contents of a destination bucket or directory match those of a source bucket or directory. This is done by copying all data from the source to the destination and deleting all other data in the destination that is not in the source. Please exercise caution when you use this option: It's possible to delete large amounts of data accidentally if, for example, you erroneously reverse source and destination. To make the local directory my-data the same as the contents of gs://mybucket/data and delete objects in the local directory that are not in gs://mybucket/data: gsutil rsync -d -r gs://mybucket/data my-data To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete objects in gs://mybucket2 that are not in gs://mybucket1: gsutil rsync -d -r gs://mybucket1 gs://mybucket2 You can also mirror data across local directories. This example will copy all objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1: gsutil rsync -d -r dir1 dir2 To mirror your content across clouds: gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket Change detection works if the other Cloud provider is using md5 or CRC32. AWS multipart upload has an incompatible checksum. As mentioned above, using -d can be dangerous because of how quickly data can be deleted. For example, if you meant to synchronize a local directory from a bucket in the cloud but instead run the command: gsutil -m rsync -r -d ./your-dir gs://your-bucket and your-dir is currently empty, you will quickly delete all of the objects in gs://your-bucket. You can also cause large amounts of data to be lost quickly by specifying a subdirectory of the destination as the source of an rsync. For example, the command: gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket would cause most or all of the objects in gs://your-bucket to be deleted (some objects may survive if there are any with names that sort lower than "data" under gs://your-bucket/data). In addition to paying careful attention to the source and destination you specify with the rsync command, there are two more safety measures you can take when using gsutil rsync -d: 1. Try running the command with the rsync -n option first, to see what it would do without actually performing the operations. For example, if you run the command: gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket it will be immediately evident that running that command without the -n option would cause many objects to be deleted. 2. Enable object versioning in your bucket, which allows you to restore objects if you accidentally delete them. For more details see `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_. <B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B> Running gsutil rsync over a directory containing operating system-specific file types (symbolic links, device files, sockets, named pipes, etc.) can cause various problems. For example, running a command like: gsutil rsync -r ./dir gs://my-bucket will cause gsutil to follow any symbolic links in ./dir, creating objects in my-bucket containing the data from the files to which the symlinks point. This can cause various problems: * If you use gsutil rsync as a simple way to backup a directory to a bucket, restoring from that bucket will result in files where the symlinks used to be. At best this is wasteful of space, and at worst it can result in outdated data or broken applications -- depending on what is consuming the symlinks. * If you use gsutil rsync over directories containing broken symlinks, gsutil rsync will abort (unless you pass the -e option). * gsutil rsync skips symlinks that point to directories. Since gsutil rsync is intended to support data operations (like moving a data set to the cloud for computational processing) and it needs to be compatible both in the cloud and across common operating systems, there are no plans for gsutil rsync to support operating system-specific file types like symlinks. We recommend that users do one of the following: * Don't use gsutil rsync over directories containing symlinks or other OS- specific file types. * Use the -e option to exclude symlinks or the -x option to exclude OS-specific file types by name. * Use a tool (such as tar) that preserves symlinks and other OS-specific file types, packaging up directories containing such files before uploading to the cloud. <B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B> While Google Cloud Storage is strongly consistent, some cloud providers only support eventual consistency. You may encounter scenarios where rsync synchronizes using stale listing data when working with these other cloud providers. For example, if you run rsync immediately after uploading an object to an eventually consistent cloud provider, the added object may not yet appear in the provider's listing. Consequently, rsync will miss adding the object to the destination. If this happens you can rerun the rsync operation again later (after the object listing has "caught up"). <B>FAILURE HANDLING</B> The rsync command retries failures when it is useful to do so, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the overall command fails. If the -C option is provided, the command instead skips failing objects and moves on. At the end of the synchronization run, if any failures were not successfully retried, the rsync command reports the count of failures and exits with non-zero status. At this point you can run the rsync command again, and gsutil attempts any remaining needed copy and/or delete operations. For more details about gsutil's retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. <B>CHANGE DETECTION ALGORITHM</B> To determine if a file or object has changed, gsutil rsync first checks whether the file modification time (mtime) of both the source and destination is available. If mtime is available at both source and destination, and the destination mtime is different than the source, or if the source and destination file size differ, gsutil rsync will update the destination. If the source is a cloud bucket and the destination is a local file system, and if mtime is not available for the source, gsutil rsync will use the time created for the cloud object as a substitute for mtime. Otherwise, if mtime is not available for either the source or the destination, gsutil rsync will fall back to using checksums. If the source and destination are both cloud buckets with checksums available, gsutil rsync will use these hashes instead of mtime. However, gsutil rsync will still update mtime at the destination if it is not present. If the source and destination have matching checksums and only the source has an mtime, gsutil rsync will copy the mtime to the destination. If neither mtime nor checksums are available, gsutil rsync will resort to comparing file sizes. Checksums will not be available when comparing composite Google Cloud Storage objects with objects at a cloud provider that does not support CRC32C (which is the only checksum available for composite objects). See 'gsutil help compose' for details about composite objects. <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (i.e., without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata (like Content-Type and Cache-Control). In contrast, when you download data from the cloud it ends up in a file, which has no associated metadata, other than file modification time (mtime). Thus, unless you have some way to hold on to or re-create that metadata, synchronizing a bucket to a directory in the local file system will not retain the metadata other than mtime. Note that by default, the gsutil rsync command does not copy the ACLs of objects being synchronized and instead will use the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the -p option. See the `Options section <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to learn how. <B>LIMITATIONS</B> 1. The gsutil rsync command will only allow non-negative file modification times to be used in its comparisons. This means gsutil rsync will resort to using checksums for any file with a timestamp before 1970-01-01 UTC. 2. The gsutil rsync command considers only the live object version in the source and destination buckets when deciding what to copy / delete. If versioning is enabled in the destination bucket then gsutil rsync's replacing or deleting objects will end up creating versions, but the command doesn't try to make any noncurrent versions match in the source and destination buckets. 3. The gsutil rsync command does not support copying special file types such as sockets, device files, named pipes, or any other non-standard files intended to represent an operating system resource. If you run gsutil rsync on a source directory that includes such files (for example, copying the root directory on Linux that includes /dev ), you should use the -x flag to exclude these files. Otherwise, gsutil rsync may fail or hang. 4. The gsutil rsync command copies changed files in their entirety and does not employ the `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_ to transfer portions of a changed file. This is because Cloud Storage objects are immutable and no facility exists to read partial object checksums or perform partial replacements. <B>OPTIONS</B> -a predef-acl Sets the specified predefined ACL on uploaded objects. See "gsutil help acls" for further details. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Also see the -p option below. -c Causes the rsync command to compute and compare checksums (instead of comparing mtime) for files if the size of source and destination match. This option increases local disk I/O and run time if either src_url or dst_url are on the local file system. -C If an error occurs, continue to attempt to copy the remaining files. If errors occurred, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rsync...". NOTE: -C only applies to the actual copying operation. If an error occurs while iterating over the files in the local directory (e.g., invalid Unicode file name) gsutil will print an error message and abort. -d Delete extra files under dst_url not found under src_url. By default extra files are not deleted. NOTE: this option can delete data quickly if you specify the wrong source/destination combination. See the help section above, "BE CAREFUL WHEN USING -d OPTION!". -e Exclude symlinks. When specified, symbolic links will be ignored. Note that gsutil does not follow directory symlinks, regardless of whether -e is specified. -i Skip copying any files that already exist at the destination, regardless of their modification time. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the -j extension list. This is useful when uploading files with compressible content (such as .js, .css, or .html files) because it saves network bandwidth while also leaving the data uncompressed in Google Cloud Storage. When you specify the -j option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the Content-Type and name of the original files. Note that if you want to use the top-level -m option to parallelize copies along with the -j/-J options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. This compression buffer size can be changed to a higher limit, e.g.: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m rsync -j html,txt /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the -j option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If you use this option and some of the source files don't compress well (e.g., that's often true of binary data), this option may result in longer uploads. -n Causes rsync to run in "dry run" mode, i.e., just outputting what would be copied or deleted without actually doing any copying/deleting. -p Causes ACLs to be preserved when objects are copied. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Thus, if the source and destination differ in size or modification time and you run gsutil rsync -p, the file will be copied and ACL preserved. However, if the source and destination don't differ in size or checksum but have different ACLs, running gsutil rsync -p will have no effect. Note that this option has performance and cost implications when using the XML API, as it requires separate HTTP calls for interacting with ACLs. The performance issue can be mitigated to some degree by using gsutil -m rsync to cause parallel synchronization. Also, this option only works if you have OWNER access to all of the objects that are copied. You can avoid the additional performance and cost of using rsync -p if you want all objects in the destination bucket to end up with the same ACL by setting a default object ACL on that bucket instead of using rsync -p. See 'gsutil help defacl'. -P Causes POSIX attributes to be preserved when objects are copied. With this feature enabled, gsutil rsync will copy fields provided by stat. These are the user ID of the owner, the group ID of the owning group, the mode (permissions) of the file, and the access/modification timestamps of the file. For downloads, these attributes will only be set if the source objects were uploaded with this flag enabled. On Windows, this flag will only set and restore access time and modification time. This is because Windows doesn't have a notion of POSIX uid/gid/mode. -R, -r The -R and -r options are synonymous. Causes directories, buckets, and bucket subdirectories to be synchronized recursively. If you neglect to use this option gsutil will make only the top-level directory in the source and destination URLs match, skipping any sub-directories. -u When a file/object is present in both the source and destination, if mtime is available for both, do not perform the copy if the destination mtime is newer. -U Skip objects with unsupported object types instead of failing. Unsupported object types are Amazon S3 Objects in the GLACIER storage class. -x pattern Causes files/objects matching pattern to be excluded, i.e., any matching files/objects are not copied or deleted. Note that the pattern is a `Python regular expression <https://docs.python.org/3/howto/regex.html>`_, not a wildcard (so, matching any string ending in "abc" would be specified using ".*abc$" rather than "*abc"). Note also that the exclude path is always relative (similar to Unix rsync or tar exclude options). For example, if you run the command: gsutil rsync -x "data.[/\\].*\\.txt$" dir gs://my-bucket it skips the file dir/data1/a.txt. You can use regex alternation to specify multiple exclusions, for example: gsutil rsync -x ".*\\.txt$|.*\\.jpg$" dir gs://my-bucket skips all .txt and .jpg files in dir. NOTE: When using the Windows cmd.exe command line interpreter, use ``^`` as an escape character instead of ``\\`` and escape the ``|`` character. When using Windows PowerShell, use ``'`` instead of ``"`` and surround the ``|`` character with ``"``. -y pattern Similar to the -x option, but the command will first skip directories/prefixes using the provided pattern and then exclude files/objects using the same pattern. This is usually much faster, but won't work as intended with negative lookahead patterns. For example, if you run the command: gsutil rsync -y "^(?!.*\\.txt$).*" dir gs://my-bucket This would first exclude all subdirectories unless they end in .txt before excluding all files except those ending in .txt. Running the same command with the -x option would result in all .txt files being included, regardless of whether they appear in subdirectories that end in .txt. """) # pylint: enable=anomalous-backslash-in-string _NA = '-' _OUTPUT_BUFFER_SIZE = 64 * 1024 _PROGRESS_REPORT_LISTING_COUNT = 10000 # Tracks files we need to clean up at end or if interrupted. Because some # files are passed to rsync's diff iterators, it is difficult to manage when # they should be closed, especially in the event that we receive a signal to # exit. Every time such a file is opened, its file object should be appended # to this list. _tmp_files = [] # pylint: disable=unused-argument def _HandleSignals(signal_num, cur_stack_frame): """Called when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.""" CleanUpTempFiles() def CleanUpTempFiles(): """Cleans up temp files. This function allows the main (RunCommand) function to clean up at end of operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary because tempfile.NamedTemporaryFile doesn't allow the created file to be re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which doesn't automatically delete temp files. """ # First pass: Close all the files. Wrapped iterators result in open file # objects for the same file, and Windows does not allow removing the file # at a given path until all its open file handles have been closed. for fileobj in _tmp_files: # Windows requires temp files to be closed before unlinking. if not fileobj.closed: fileobj.close() # Second pass: Remove each file, skipping duplicates that have already been # removed. for fileobj in _tmp_files: if os.path.isfile(fileobj.name): try: os.unlink(fileobj.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to close and delete temp file "%s". Got an error:\n%s', fileobj.name, e) def _DiffToApplyArgChecker(command_instance, diff_to_apply): """Arg checker that skips symlinks if -e flag specified.""" if (diff_to_apply.diff_action == DiffAction.REMOVE or not command_instance.exclude_symlinks): # No src URL is populated for REMOVE actions. return True exp_src_url = StorageUrlFromString(diff_to_apply.src_url_str) if exp_src_url.IsFileUrl() and os.path.islink(exp_src_url.object_name): command_instance.logger.info('Skipping symbolic link %s...', exp_src_url) return False return True def _ComputeNeededFileChecksums(logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5): """Computes any file checksums needed by _CompareObjects. Args: logger: logging.logger for outputting log messages. src_url_str: Source URL string. src_size: Source size src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: (src_crc32c, src_md5, dst_crc32c, dst_md5) """ src_url = StorageUrlFromString(src_url_str) dst_url = StorageUrlFromString(dst_url_str) if src_url.IsFileUrl(): if dst_crc32c != _NA or dst_url.IsFileUrl(): if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif dst_md5 != _NA or dst_url.IsFileUrl(): if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_md5 = CalculateB64EncodedMd5FromContents(fp) if dst_url.IsFileUrl(): if src_crc32c != _NA: if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif src_md5 != _NA: if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_md5 = CalculateB64EncodedMd5FromContents(fp) return (src_crc32c, src_md5, dst_crc32c, dst_md5) def _ListUrlRootFunc(cls, args_tuple, thread_state=None): """Worker function for listing files/objects under to be sync'd. Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We sort the listed URLs because we don't want to depend on consistent sort order across file systems and cloud providers. Args: cls: Command instance. args_tuple: (base_url_str, out_file_name, desc), where base_url_str is top-level URL string to list; out_filename is name of file to which sorted output should be written; desc is 'source' or 'destination'. thread_state: gsutil Cloud API instance to use. """ gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) (base_url_str, out_filename, desc) = args_tuple # We sort while iterating over base_url_str, allowing parallelism of batched # sorting with collecting the listing. out_file = io.open(out_filename, mode='w', encoding=constants.UTF8) try: _BatchSort(_FieldedListingIterator(cls, gsutil_api, base_url_str, desc), out_file) except Exception as e: # pylint: disable=broad-except # Abandon rsync if an exception percolates up to this layer - retryable # exceptions are handled in the lower layers, so we got a non-retryable # exception (like 404 bucket not found) and proceeding would either be # futile or could result in data loss - for example: # gsutil rsync -d gs://non-existent-bucket ./localdir # would delete files from localdir. cls.logger.error('Caught non-retryable exception while listing %s: %s' % (base_url_str, e)) # Also print the full stack trace in debugging mode. This makes debugging # a bit easier. cls.logger.debug(traceback.format_exc()) cls.non_retryable_listing_failures = 1 out_file.close() def _LocalDirIterator(base_url): """A generator that yields a BLR for each file in a local directory. We use this function instead of WildcardIterator for listing a local directory without recursion, because the glob.globi implementation called by WildcardIterator skips "dot" files (which we don't want to do when synchronizing to or from a local directory). Args: base_url: URL for the directory over which to iterate. Yields: BucketListingObject for each file in the directory. """ for filename in os.listdir(base_url.object_name): filename = os.path.join(base_url.object_name, filename) if os.path.isfile(filename): yield BucketListingObject(StorageUrlFromString(filename), None) def _FieldedListingIterator(cls, gsutil_api, base_url_str, desc): """Iterator over base_url_str formatting output per _BuildTmpOutputLine. Args: cls: Command instance. gsutil_api: gsutil Cloud API instance to use for bucket listing. base_url_str: The top-level URL string over which to iterate. desc: 'source' or 'destination'. Yields: Output line formatted per _BuildTmpOutputLine. """ base_url = StorageUrlFromString(base_url_str) if base_url.scheme == 'file' and not cls.recursion_requested: iterator = _LocalDirIterator(base_url) else: if cls.recursion_requested: wildcard = '%s/**' % base_url_str.rstrip('/\\') else: wildcard = '%s/*' % base_url_str.rstrip('/\\') fields = [ 'crc32c', 'md5Hash', 'name', 'size', 'timeCreated', 'metadata/%s' % MTIME_ATTR, ] if cls.preserve_posix_attrs: fields.extend([ 'metadata/%s' % ATIME_ATTR, 'metadata/%s' % MODE_ATTR, 'metadata/%s' % GID_ATTR, 'metadata/%s' % UID_ATTR, ]) exclude_tuple = ( base_url, cls.exclude_dirs, cls.exclude_pattern) if cls.exclude_pattern is not None else None iterator = CreateWildcardIterator( wildcard, gsutil_api, project_id=cls.project_id, exclude_tuple=exclude_tuple, ignore_symlinks=cls.exclude_symlinks, logger=cls.logger).IterObjects( # Request just the needed fields, to reduce bandwidth usage. bucket_listing_fields=fields) i = 0 for blr in iterator: # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. # We need to filter them out here, otherwise if the user tries to rsync # from GCS to a local directory it will result in a directory/file # conflict (e.g., trying to download an object called "mydata/" where the # local directory "mydata" exists). url = blr.storage_url if IsCloudSubdirPlaceholder(url, blr=blr): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. continue if (cls.exclude_symlinks and url.IsFileUrl() and os.path.islink(url.object_name)): continue if cls.exclude_pattern: # The wildcard_iterator may optionally use the exclude pattern to exclude # directories while this section excludes individual files. str_to_check = url.url_string[len(base_url.url_string):] if str_to_check.startswith(url.delim): str_to_check = str_to_check[1:] if cls.exclude_pattern.match(str_to_check): continue i += 1 if i % _PROGRESS_REPORT_LISTING_COUNT == 0: cls.logger.info('At %s listing %d...', desc, i) yield _BuildTmpOutputLine(blr) def _BuildTmpOutputLine(blr): """Builds line to output to temp file for given BucketListingRef. Args: blr: The BucketListingRef. Returns: The output line, formatted as _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp> gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that aren't composite objects. A missing field is populated with '-', or -1 in the case of atime/mtime/time_created. """ atime = NA_TIME crc32c = _NA gid = NA_ID md5 = _NA mode = NA_MODE mtime = NA_TIME time_created = NA_TIME uid = NA_ID url = blr.storage_url if url.IsFileUrl(): mode, _, _, _, uid, gid, size, atime, mtime, _ = os.stat(url.object_name) # atime/mtime can be a float, so it needs to be converted to a long. atime = long(atime) mtime = long(mtime) mode = ConvertModeToBase8(mode) # Don't use atime / mtime with times older than 1970-01-01 UTC. if atime < 0: atime = NA_TIME if mtime < 0: mtime = NA_TIME elif url.IsCloudUrl(): size = blr.root_object.size if blr.root_object.metadata is not None: found_m, mtime_str = GetValueFromObjectCustomMetadata( blr.root_object, MTIME_ATTR, NA_TIME) try: # The mtime value can be changed in the online console, this performs a # sanity check and sets the mtime to NA_TIME if it fails. mtime = long(mtime_str) if found_m and mtime <= NA_TIME: WarnNegativeAttribute('mtime', url.url_string) if mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', url.url_string) except ValueError: # Since mtime is a string, catch the case where it can't be cast as a # long. WarnInvalidValue('mtime', url.url_string) mtime = NA_TIME posix_attrs = DeserializeFileAttributesFromObjectMetadata( blr.root_object, url.url_string) mode = posix_attrs.mode.permissions atime = posix_attrs.atime uid = posix_attrs.uid gid = posix_attrs.gid # Sanitize the timestamp returned, and put it in UTC format. For more # information see the UTC class in gslib/util.py. time_created = ConvertDatetimeToPOSIX(blr.root_object.timeCreated) crc32c = blr.root_object.crc32c or _NA md5 = blr.root_object.md5Hash or _NA else: raise CommandException('Got unexpected URL type (%s)' % url.scheme) attrs = [ _EncodeUrl(url.url_string), # binary str in py2 / unicode str py 3 size, # int time_created, # int atime, # long mtime, # long mode, # int uid, # int gid, # int crc32c, # unicode md5, # unicode ] attrs = [six.ensure_text(str(i)) for i in attrs] return ' '.join(attrs) + '\n' def _EncodeUrl(url_string): """Encodes url_str with quote plus encoding and UTF8 character encoding. We use this for all URL encodings. Args: url_string (unicode): String URL to encode. Returns: (str) A string encoded using urllib's `quote_plus()` method. """ # N.B.: `quote_plus()` raises an error for unicode characters like è if you # don't pass it the language-appropriate string type. If you pass it `unicode` # in Python 2 or `bytes` in Python 3, it leads to surprising behavior for text # containing unicode chars. url_string = six.ensure_str(url_string) return urllib.parse.quote_plus(url_string, safe=b'~') def _DecodeUrl(enc_url_string): """Inverts encoding from `_EncodeUrl()`. Args: enc_url_string (str): String containing UTF-8-decodable characters that were encoded using urllib's `quote_plus()`. Returns: (unicode) A decoded URL. """ url = urllib.parse.unquote_plus(enc_url_string) if six.PY2: url = url.decode(constants.UTF8) return url # pylint: disable=bare-except def _BatchSort(in_iter, out_file): """Sorts input lines from in_iter and outputs to out_file. Sorts in batches as input arrives, so input file does not need to be loaded into memory all at once. Derived from Python Recipe 466302: Sorting big files the Python 2.4 way by Nicolas Lehuen. Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line when we could just sort on the first record (URL); but the sort order is identical either way. Args: in_iter: Input iterator. out_file: Output file. """ # Note: If chunk_files gets very large we can run out of open FDs. See .boto # file comments about rsync_buffer_lines. If increasing rsync_buffer_lines # doesn't suffice (e.g., for someone synchronizing with a really large # bucket), an option would be to make gsutil merge in passes, never # opening all chunk files simultaneously. buffer_size = config.getint('GSUtil', 'rsync_buffer_lines', 32000) chunk_files = [] try: while True: current_chunk = sorted(islice(in_iter, buffer_size)) if not current_chunk: break output_chunk = io.open('%s-%06i' % (out_file.name, len(chunk_files)), mode='w+', encoding=constants.UTF8) chunk_files.append(output_chunk) output_chunk.write(six.text_type(''.join(current_chunk))) output_chunk.flush() output_chunk.seek(0) out_file.writelines(heapq.merge(*chunk_files)) except IOError as e: if e.errno == errno.EMFILE: raise CommandException('\n'.join( textwrap.wrap( 'Synchronization failed because too many open file handles were ' 'needed while building synchronization state. Please see the ' 'comments about rsync_buffer_lines in your .boto config file for a ' 'possible way to address this problem.'))) raise finally: for chunk_file in chunk_files: try: chunk_file.close() os.remove(chunk_file.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to remove rsync chunk file "%s". Got an error:\n%s', chunk_file.name, e) class _DiffIterator(object): """Iterator yielding sequence of RsyncDiffToApply objects.""" def __init__(self, command_obj, base_src_url, base_dst_url): self.command_obj = command_obj self.compute_file_checksums = command_obj.compute_file_checksums self.delete_extras = command_obj.delete_extras self.recursion_requested = command_obj.recursion_requested self.logger = self.command_obj.logger self.base_src_url = base_src_url self.base_dst_url = base_dst_url self.preserve_posix = command_obj.preserve_posix_attrs self.skip_old_files = command_obj.skip_old_files self.ignore_existing = command_obj.ignore_existing self.logger.info('Building synchronization state...') # Files to track src and dst state should be created in the system's # preferred temp directory so that they are eventually cleaned up if our # cleanup callback is interrupted. temp_src_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-src-', delete=False) temp_dst_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-dst-', delete=False) self.sorted_list_src_file_name = temp_src_file.name self.sorted_list_dst_file_name = temp_dst_file.name _tmp_files.append(temp_src_file) _tmp_files.append(temp_dst_file) # Close the files, but don't delete them. Because Windows does not allow # a temporary file to be reopened until it's been closed, we close the # files before proceeding. This allows each step below to open the file at # the specified path, perform I/O, and close it so that the next step may # do the same thing. temp_src_file.close() temp_dst_file.close() # Build sorted lists of src and dst URLs in parallel. To do this, pass # args to _ListUrlRootFunc as tuple (base_url_str, out_filename, desc) # where base_url_str is the starting URL string for listing. args_iter = iter([ ( self.base_src_url.url_string, self.sorted_list_src_file_name, 'source', ), ( self.base_dst_url.url_string, self.sorted_list_dst_file_name, 'destination', ), ]) # Contains error message from non-retryable listing failure. command_obj.non_retryable_listing_failures = 0 shared_attrs = ['non_retryable_listing_failures'] command_obj.Apply( _ListUrlRootFunc, args_iter, _RootListingExceptionHandler, shared_attrs, arg_checker=DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SPEED, fail_on_error=True) if command_obj.non_retryable_listing_failures: raise CommandException('Caught non-retryable exception - aborting rsync') # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open(self.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open(self.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) if (base_src_url.IsCloudUrl() and base_dst_url.IsFileUrl() and self.preserve_posix): self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self._ValidateObjectAccess() # Reset our file pointers to the beginning. self.sorted_list_src_file.seek(0) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) def _ValidateObjectAccess(self): """Validates that the user won't lose access to the files if copied. Iterates over the src file list to check if access will be maintained. If at any point we would orphan a file, a list of errors is compiled and logged with an exception raised to the user. """ errors = collections.deque() for src_url in self.sorted_src_urls_it: src_url_str, _, _, _, _, src_mode, src_uid, src_gid, _, _ = ( self._ParseTmpFileLine(src_url)) valid, err = ValidateFilePermissionAccess(src_url_str, uid=src_uid, gid=src_gid, mode=src_mode) if not valid: errors.append(err) if errors: for err in errors: self.logger.critical(err) raise CommandException('This sync will orphan file(s), please fix their ' 'permissions before trying again.') def _ParseTmpFileLine(self, line): """Parses output from _BuildTmpOutputLine. Parses into tuple: (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) where crc32c and/or md5 can be _NA and atime/mtime/time_created can be NA_TIME. Args: line: The line to parse. Returns: Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) """ (encoded_url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) = line.rsplit(None, 9) return ( _DecodeUrl(encoded_url), int(size), long(time_created), long(atime), long(mtime), int(mode), int(uid), int(gid), crc32c, md5.strip(), ) def _WarnIfMissingCloudHash(self, url_str, crc32c, md5): """Warns if given url_str is a cloud URL and is missing both crc32c and md5. Args: url_str: Destination URL string. crc32c: Destination CRC32c. md5: Destination MD5. Returns: True if issued warning. """ # One known way this can currently happen is when rsync'ing objects larger # than 5 GB from S3 (for which the etag is not an MD5). if (StorageUrlFromString(url_str).IsCloudUrl() and crc32c == _NA and md5 == _NA): self.logger.warn( 'Found no hashes to validate %s. Integrity cannot be assured without ' 'hashes.', url_str) return True return False def _CompareObjects( self, src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5, ): """Returns whether src should replace dst object, and if mtime is present. Uses mtime, size, or whatever checksums are available. Args: src_url_str: Source URL string. src_size: Source size. src_mtime: Source modification time. src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size. dst_mtime: Destination modification time. dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: A 3-tuple indicating if src should replace dst, and if src and dst have mtime. """ # Note: This function is called from __iter__, which is called from the # Command.Apply driver. Thus, all checksum computation will be run in a # single thread, which is good (having multiple threads concurrently # computing checksums would thrash the disk). # # Comparison Hierarchy: # 1. mtime # 2. md5/crc32c hashes (if available) # 3. size has_src_mtime = src_mtime > NA_TIME has_dst_mtime = dst_mtime > NA_TIME use_hashes = (self.compute_file_checksums or (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsCloudUrl())) if self.ignore_existing: return False, has_src_mtime, has_dst_mtime if (self.skip_old_files and has_src_mtime and has_dst_mtime and src_mtime < dst_mtime): return False, has_src_mtime, has_dst_mtime if not use_hashes and has_src_mtime and has_dst_mtime: return (src_mtime != dst_mtime or src_size != dst_size, has_src_mtime, has_dst_mtime) if src_size != dst_size: return True, has_src_mtime, has_dst_mtime src_crc32c, src_md5, dst_crc32c, dst_md5 = _ComputeNeededFileChecksums( self.logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5, ) if src_md5 != _NA and dst_md5 != _NA: self.logger.debug('Comparing md5 for %s and %s', src_url_str, dst_url_str) return src_md5 != dst_md5, has_src_mtime, has_dst_mtime if src_crc32c != _NA and dst_crc32c != _NA: self.logger.debug('Comparing crc32c for %s and %s', src_url_str, dst_url_str) return src_crc32c != dst_crc32c, has_src_mtime, has_dst_mtime if not self._WarnIfMissingCloudHash(src_url_str, src_crc32c, src_md5): self._WarnIfMissingCloudHash(dst_url_str, dst_crc32c, dst_md5) # Without checksums or mtime to compare we depend only on basic size # comparison. return False, has_src_mtime, has_dst_mtime def __iter__(self): """Iterates over src/dst URLs and produces a RsyncDiffToApply sequence. Yields: The RsyncDiffToApply. """ # Strip trailing slashes, if any, so we compute tail length against # consistent position regardless of whether trailing slashes were included # or not in URL. base_src_url_len = len(self.base_src_url.url_string.rstrip('/\\')) base_dst_url_len = len(self.base_dst_url.url_string.rstrip('/\\')) out_of_src_items = False src_url_str = dst_url_str = None # Invariant: After each yield, the URLs in src_url_str, dst_url_str, # self.sorted_src_urls_it, and self.sorted_dst_urls_it are not yet # processed. Each time we encounter None in src_url_str or dst_url_str we # populate from the respective iterator, and we reset one or the other value # to None after yielding an action that disposes of that URL. while True: if src_url_str is None: if self.sorted_src_urls_it.IsEmpty(): out_of_src_items = True else: (src_url_str, src_size, src_time_created, src_atime, src_mtime, src_mode, src_uid, src_gid, src_crc32c, src_md5) = (self._ParseTmpFileLine(next(self.sorted_src_urls_it))) posix_attrs = POSIXAttributes(atime=src_atime, mtime=src_mtime, uid=src_uid, gid=src_gid, mode=src_mode) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). src_url_str_to_check = _EncodeUrl( src_url_str[base_src_url_len:].replace('\\', '/')) dst_url_str_would_copy_to = copy_helper.ConstructDstUrl( src_url=self.base_src_url, exp_src_url=StorageUrlFromString(src_url_str), src_url_names_container=True, have_multiple_srcs=True, has_multiple_top_level_srcs=False, exp_dst_url=self.base_dst_url, have_existing_dest_subdir=False, recursion_requested=self.recursion_requested).url_string if dst_url_str is None: if not self.sorted_dst_urls_it.IsEmpty(): # We don't need time created at the destination. (dst_url_str, dst_size, _, dst_atime, dst_mtime, dst_mode, dst_uid, dst_gid, dst_crc32c, dst_md5) = self._ParseTmpFileLine(next(self.sorted_dst_urls_it)) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). dst_url_str_to_check = _EncodeUrl( dst_url_str[base_dst_url_len:].replace('\\', '/')) # Only break once we've attempted to populate {str,dst}_url_to_check and # we know we're out of src objects. if out_of_src_items: break # We're guaranteed to have a value for src_url_str_to_check here, but may # be out of dst objects. if (dst_url_str is None or src_url_str_to_check < dst_url_str_to_check): # There's no dst object corresponding to src object, so copy src to dst. yield RsyncDiffToApply(src_url_str, dst_url_str_would_copy_to, posix_attrs, DiffAction.COPY, src_size) src_url_str = None elif src_url_str_to_check > dst_url_str_to_check: # dst object without a corresponding src object, so remove dst if -d # option was specified. if self.delete_extras: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) dst_url_str = None else: # There is a dst object corresponding to src object, so check if objects # match. if (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsFileUrl() and src_mtime == NA_TIME): src_mtime = src_time_created should_replace, has_src_mtime, has_dst_mtime = (self._CompareObjects( src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5)) if should_replace: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.COPY, src_size) elif self.preserve_posix: posix_attrs, needs_update = NeedsPOSIXAttributeUpdate( src_atime, dst_atime, src_mtime, dst_mtime, src_uid, dst_uid, src_gid, dst_gid, src_mode, dst_mode) if needs_update: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.POSIX_SRC_TO_DST, src_size) elif has_src_mtime and not has_dst_mtime: # File/object at destination matches source but is missing mtime # attribute at destination. yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.MTIME_SRC_TO_DST, src_size) # else: we don't need to copy the file from src to dst since they're # the same files. # Advance to the next two objects. src_url_str = None dst_url_str = None if not self.delete_extras: return # If -d option was specified any files/objects left in dst iteration should # be removed. if dst_url_str: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) for line in self.sorted_dst_urls_it: (dst_url_str, _, _, _, _, _, _, _, _, _) = self._ParseTmpFileLine(line) yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) class _SeekAheadDiffIterator(object): """Wraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.""" def __init__(self, cloned_diff_iterator): self.cloned_diff_iterator = cloned_diff_iterator def __iter__(self): for diff_to_apply in self.cloned_diff_iterator: bytes_to_copy = diff_to_apply.copy_size or 0 if (diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST or diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST): # Assume MTIME_SRC_TO_DST and POSIX_SRC_TO_DST are metadata-only # copies. However, if the user does not have OWNER permission on # an object, the data must be re-sent, and this function will # underestimate the amount of bytes that rsync must copy. bytes_to_copy = 0 yield SeekAheadResult(data_bytes=bytes_to_copy) class _AvoidChecksumAndListingDiffIterator(_DiffIterator): """Iterator initialized from an existing _DiffIterator. This iterator yields RsyncDiffToApply objects used to estimate the total work that will be performed by the DiffIterator, while avoiding expensive computation. """ # pylint: disable=super-init-not-called def __init__(self, initialized_diff_iterator): # Intentionally don't call the _DiffIterator constructor. This class # reuses the initialized_diff_iterator values to avoid unnecessary # computation, and inherits the __iter__ function. # We're providing an estimate, so avoid computing checksums even though # that may cause our estimate to be off. self.compute_file_checksums = False self.delete_extras = initialized_diff_iterator.delete_extras self.recursion_requested = initialized_diff_iterator.delete_extras # TODO: Add a test that mocks the appropriate values in RsyncFunc and # ensure that running this iterator succeeds. self.preserve_posix = False # This iterator shouldn't output any log messages. self.logger = logging.getLogger('dummy') self.base_src_url = initialized_diff_iterator.base_src_url self.base_dst_url = initialized_diff_iterator.base_dst_url self.skip_old_files = initialized_diff_iterator.skip_old_files self.ignore_existing = initialized_diff_iterator.ignore_existing # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open( initialized_diff_iterator.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open( initialized_diff_iterator.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) # pylint: enable=super-init-not-called def _RsyncFunc(cls, diff_to_apply, thread_state=None): """Worker function for performing the actual copy and remove operations.""" gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) dst_url_str = diff_to_apply.dst_url_str dst_url = StorageUrlFromString(dst_url_str) posix_attrs = diff_to_apply.src_posix_attrs if diff_to_apply.diff_action == DiffAction.REMOVE: if cls.dryrun: cls.logger.info('Would remove %s', dst_url) else: cls.logger.info('Removing %s', dst_url) if dst_url.IsFileUrl(): try: os.unlink(dst_url.object_name) except FileNotFoundError: # Missing file errors occur occasionally with .gstmp files # and can be ignored for deletes. cls.logger.debug('%s was already removed', dst_url) pass else: try: gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=dst_url.generation, provider=dst_url.scheme) except NotFoundException: # If the object happened to be deleted by an external process, this # is fine because it moves us closer to the desired state. pass elif diff_to_apply.diff_action == DiffAction.COPY: src_url_str = diff_to_apply.src_url_str src_url = StorageUrlFromString(src_url_str) if cls.dryrun: if src_url.IsFileUrl(): # Try to open the local file to detect errors that would occur in # non-dry-run mode. try: with open(src_url.object_name, 'rb') as _: pass except Exception as e: # pylint: disable=broad-except cls.logger.info('Could not open %s' % src_url.object_name) raise cls.logger.info('Would copy %s to %s', src_url, dst_url) else: try: src_obj_metadata = None if src_url.IsCloudUrl(): src_generation = GenerationFromUrlAndString(src_url, src_url.generation) src_obj_metadata = gsutil_api.GetObjectMetadata( src_url.bucket_name, src_url.object_name, generation=src_generation, provider=src_url.scheme, fields=cls.source_metadata_fields) if ObjectIsGzipEncoded(src_obj_metadata): cls.logger.info( '%s has a compressed content-encoding, so it will be ' 'decompressed upon download; future executions of gsutil rsync ' 'with this source object will always download it. If you wish ' 'to synchronize such an object efficiently, compress the ' 'source objects in place before synchronizing them, rather ' 'than (for example) using gsutil cp -Z to compress them ' 'on-the-fly (which results in compressed content-encoding).' % src_url) else: # src_url.IsFileUrl() src_obj_metadata = apitools_messages.Object() # getmtime can return a float, so it needs to be converted to long. if posix_attrs.mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', src_url.url_string) if src_url.IsFifo() or src_url.IsStream(): type_text = 'Streams' if src_url.IsStream() else 'Named pipes' cls.logger.warn( 'WARNING: %s are not supported by gsutil rsync and ' 'will likely fail. Use the -x option to exclude %s by name.', type_text, src_url.url_string) if src_obj_metadata.metadata: custom_metadata = src_obj_metadata.metadata else: custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata( posix_attrs, custom_metadata, preserve_posix=cls.preserve_posix_attrs) tmp_obj_metadata = apitools_messages.Object() tmp_obj_metadata.metadata = custom_metadata CopyCustomMetadata(tmp_obj_metadata, src_obj_metadata, override=True) copy_result = copy_helper.PerformCopy( cls.logger, src_url, dst_url, gsutil_api, cls, _RsyncExceptionHandler, src_obj_metadata=src_obj_metadata, headers=cls.headers, is_rsync=True, gzip_encoded=cls.gzip_encoded, gzip_exts=cls.gzip_exts, preserve_posix=cls.preserve_posix_attrs) if copy_result is not None: (_, bytes_transferred, _, _) = copy_result with cls.stats_lock: cls.total_bytes_transferred += bytes_transferred except SkipUnsupportedObjectError as e: cls.logger.info('Skipping item %s with unsupported object type %s', src_url, e.unsupported_type) elif diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set mtime for %s', dst_url) else: cls.logger.info('Copying mtime from src to dst for %s', dst_url.url_string) mtime = posix_attrs.mtime obj_metadata = apitools_messages.Object() obj_metadata.metadata = CreateCustomMetadata({MTIME_ATTR: mtime}) if dst_url.IsCloudUrl(): dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) try: # Assume we have permission, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # We don't have permission to patch apparently, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: ParseAndSetPOSIXAttributes(dst_url.object_name, obj_metadata, preserve_posix=cls.preserve_posix_attrs) elif diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set POSIX attributes for %s', dst_url) else: cls.logger.info('Copying POSIX attributes from src to dst for %s', dst_url.url_string) obj_metadata = apitools_messages.Object() obj_metadata.metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, obj_metadata.metadata, preserve_posix=True) if dst_url.IsCloudUrl(): dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) dst_obj_metadata = gsutil_api.GetObjectMetadata( dst_url.bucket_name, dst_url.object_name, generation=dst_generation, provider=dst_url.scheme, fields=['acl']) try: # Assume we have ownership, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # Apparently we don't have object ownership, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: raise CommandException('Got unexpected DiffAction (%d)' % diff_to_apply.diff_action) def _RootListingExceptionHandler(cls, e): """Simple exception handler for exceptions during listing URLs to sync.""" cls.logger.error(str(e)) def _RsyncExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while syncing:\n%s\n', traceback.format_exc()) class RsyncCommand(Command): """Implementation of gsutil rsync command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rsync', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=2, supported_sub_args='a:cCdenpPriRuUx:y:j:J', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudOrFileURLsArgument(2)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rsync', help_name_aliases=['sync', 'synchronize'], help_type='command_help', help_one_line_summary='Synchronize content of two buckets/directories', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_command = ['storage', 'rsync'] flag_keys = [flag for flag, _ in self.sub_opts] if '-e' not in flag_keys: gcloud_command += ['--no-ignore-symlinks'] self.logger.warn( 'By default, gsutil copies file symlinks, but, by default, this' ' command (run via the gcloud storage shim) does not copy any' ' symlinks.') if '-P' in flag_keys: _, (source_path, destination_path) = self.ParseSubOpts( should_update_sub_opts_and_args=False) if (StorageUrlFromString(source_path).IsCloudUrl() and StorageUrlFromString(destination_path).IsFileUrl()): self.logger.warn( 'For preserving POSIX with rsync downloads, gsutil aborts if a' ' single download will result in invalid destination POSIX.' ' However, this command (run via the gcloud storage shim) will' ' skip invalid copies and still perform valid copies.') gcloud_storage_map = GcloudStorageMap( gcloud_command=gcloud_command, flag_map={ '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--checksums-only'), '-C': GcloudStorageFlag('--continue-on-error'), '-d': GcloudStorageFlag('--delete-unmatched-destination-objects'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-i': GcloudStorageFlag('--no-clobber'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-n': GcloudStorageFlag('--dry-run'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), '-U': GcloudStorageFlag('--skip-unsupported'), '-u': GcloudStorageFlag('--skip-if-dest-has-newer-mtime'), '-x': GcloudStorageFlag('--exclude'), }, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _InsistContainer(self, url_str, treat_nonexistent_object_as_subdir): """Sanity checks that URL names an existing container. Args: url_str: URL string to check. treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. Returns: URL for checked string. Raises: CommandException if url_str doesn't name an existing container. """ (url, have_existing_container) = copy_helper.ExpandUrlToSingleBlr( url_str, self.gsutil_api, self.project_id, treat_nonexistent_object_as_subdir, logger=self.logger) if not have_existing_container: raise CommandException( 'arg (%s) does not name a directory, bucket, or bucket subdir.\n' 'If there is an object with the same path, please add a trailing\n' 'slash to specify the directory.' % url_str) return url def RunCommand(self): """Command entry point for the rsync command.""" self._ParseOpts() self.total_bytes_transferred = 0 # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() if not UsingCrcmodExtension(): if self.compute_file_checksums: self.logger.warn(SLOW_CRCMOD_WARNING) else: self.logger.warn(SLOW_CRCMOD_RSYNC_WARNING) src_url = self._InsistContainer(self.args[0], False) dst_url = self._InsistContainer(self.args[1], True) is_daisy_chain = (src_url.IsCloudUrl() and dst_url.IsCloudUrl() and src_url.scheme != dst_url.scheme) LogPerformanceSummaryParams(has_file_src=src_url.IsFileUrl(), has_cloud_src=src_url.IsCloudUrl(), has_file_dst=dst_url.IsFileUrl(), has_cloud_dst=dst_url.IsCloudUrl(), is_daisy_chain=is_daisy_chain, uses_fan=self.parallel_operations, provider_types=[src_url.scheme, dst_url.scheme]) self.source_metadata_fields = GetSourceFieldsNeededForCopy( dst_url.IsCloudUrl(), self.skip_unsupported_objects, self.preserve_acl, is_rsync=True, preserve_posix=self.preserve_posix_attrs) # Tracks if any copy or rm operations failed. self.op_failure_count = 0 # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _HandleSignals) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=self.ParallelOverrideReason.SPEED, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, worker_count=process_count * thread_count, ) # Perform sync requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. diff_iterator = _DiffIterator(self, src_url, dst_url) # For estimation purposes, create a SeekAheadIterator based on the # source and destination files generated when creating the diff iterator. # This iteration should avoid expensive operations like file checksumming. seek_ahead_iterator = _SeekAheadDiffIterator( _AvoidChecksumAndListingDiffIterator(diff_iterator)) self.logger.info('Starting synchronization...') start_time = time.time() try: self.Apply(_RsyncFunc, diff_iterator, _RsyncExceptionHandler, shared_attrs, arg_checker=_DiffToApplyArgChecker, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) finally: CleanUpTempFiles() end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( avg_throughput=self.total_bytes_per_second, total_elapsed_time=self.total_elapsed_time, total_bytes_transferred=self.total_bytes_transferred) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be copied/removed.' % (self.op_failure_count, plural_str, plural_str)) def _ParseOpts(self): # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False self.delete_extras = False self.preserve_acl = False self.preserve_posix_attrs = False self.compute_file_checksums = False self.dryrun = False self.exclude_dirs = False self.exclude_pattern = None self.skip_old_files = False self.ignore_existing = False self.skip_unsupported_objects = False # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. gzip_encoded = False gzip_arg_exts = None gzip_arg_all = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-c': self.compute_file_checksums = True # Note: In gsutil cp command this is specified using -c but here we use # -C so we can use -c for checksum arg (to be consistent with Unix rsync # command options). elif o == '-C': self.continue_on_error = True elif o == '-d': self.delete_extras = True elif o == '-e': self.exclude_symlinks = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-n': self.dryrun = True elif o == '-p': self.preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True if not IS_WINDOWS: InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-u': self.skip_old_files = True elif o == '-i': self.ignore_existing = True elif o == '-U': self.skip_unsupported_objects = True elif o == '-x' or o == '-y': if o == '-y': self.exclude_dirs = True if not a: raise CommandException('Invalid blank exclude filter') try: self.exclude_pattern = re.compile(a) except re.error: raise CommandException('Invalid exclude filter (%s)' % a) if self.preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if gzip_arg_exts and gzip_arg_all: raise CommandException( 'Specifying both the -j and -J options together is invalid.') self.gzip_encoded = gzip_encoded self.gzip_exts = gzip_arg_exts or gzip_arg_all return CreateCopyHelperOpts( canned_acl=canned_acl, preserve_acl=self.preserve_acl, skip_unsupported_objects=self.skip_unsupported_objects) kms.py 0000644 00000051363 15025166704 0005726 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the kms command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import textwrap from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.kms_api import KmsApi from gslib.project_id import PopulateProjectId from gslib.third_party.kms_apitools.cloudkms_v1_messages import Binding from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.encryption_helper import ValidateCMEK from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _AUTHORIZE_SYNOPSIS = """ gsutil kms authorize [-p <proj_id>] -k <kms_key> """ _ENCRYPTION_SYNOPSIS = """ gsutil kms encryption [(-d|[-k <kms_key>])] [-w] gs://<bucket_name>... """ _SERVICEACCOUNT_SYNOPSIS = """ gsutil kms serviceaccount [-p <proj_id>] """ _SYNOPSIS = (_AUTHORIZE_SYNOPSIS + _ENCRYPTION_SYNOPSIS.lstrip('\n') + _SERVICEACCOUNT_SYNOPSIS.lstrip('\n') + '\n') # pylint: disable=line-too-long _AUTHORIZE_DESCRIPTION = """ <B>AUTHORIZE</B> The authorize sub-command checks that the default (or supplied) project has a Cloud Storage service agent created for it, and if not, it creates one. It then adds appropriate encrypt/decrypt permissions to Cloud KMS resources such that the service agent can write and read Cloud KMS-encrypted objects in buckets associated with the service agent's project. <B>AUTHORIZE EXAMPLES</B> Authorize "my-project" to use a Cloud KMS key: gsutil kms authorize -p my-project \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key <B>AUTHORIZE OPTIONS</B> -k <key> The path to the KMS key to use. The path has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -p <project> The ID or number of the project being authorized to use the Cloud KMS key. If this flag is not included, your default project is authorized. """ _ENCRYPTION_DESCRIPTION = """ <B>ENCRYPTION</B> The encryption sub-command is used to set, display, or clear a bucket's default KMS key, which is used to encrypt newly-written objects if no other key is specified. <B>ENCRYPTION EXAMPLES</B> Set the default KMS key for my-bucket: gsutil kms encryption \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key \\ gs://my-bucket Show the default KMS key for my-bucket, if one is set: gsutil kms encryption gs://my-bucket Clear the default KMS key so newly-written objects are not encrypted using it: gsutil kms encryption -d gs://my-bucket Once you clear the default KMS key, newly-written objects are encrypted with Google-managed encryption keys by default. <B>ENCRYPTION OPTIONS</B> -k <key> Set the default KMS key for my-bucket using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -w (used with -k key) Display a warning rather than failing if gsutil is unable to verify that the specified key contains the correct IAM bindings for encryption/decryption. This is useful for users that do not have getIamPolicy permission but know that the key has the correct IAM policy for encryption in the user's project. -d Clear the default KMS key. """ # pylint: enable=line-too-long _SERVICEACCOUNT_DESCRIPTION = """ <B>SERVICEACCOUNT</B> The serviceaccount sub-command displays the Cloud Storage service agent that is used to perform Cloud KMS operations against your default project (or a supplied project). <B>SERVICEACCOUNT EXAMPLES</B> Show the service account for my-project: gsutil kms serviceaccount -p my-project <B>SERVICEACCOUNT OPTIONS</B> -p <project> The ID or number of the project whose Cloud Storage service agent is being requested. If this flag is not included, your default project is used. """ _DESCRIPTION = """ The kms command is used to configure Google Cloud Storage and Cloud KMS resources to support encryption of Cloud Storage objects with `Cloud KMS keys <https://cloud.google.com/storage/docs/encryption/customer-managed-keys>`_. The kms command has three sub-commands that deal with configuring Cloud Storage's integration with Cloud KMS: ``authorize``, ``encryption``, and ``serviceaccount``. Before using this command, read the `prerequisites <https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#prereqs>`_. for using Cloud KMS with Cloud Storage. """ + (_AUTHORIZE_DESCRIPTION + _ENCRYPTION_DESCRIPTION + _SERVICEACCOUNT_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _authorize_help_text = CreateHelpText(_AUTHORIZE_SYNOPSIS, _AUTHORIZE_DESCRIPTION) _encryption_help_text = CreateHelpText(_ENCRYPTION_SYNOPSIS, _ENCRYPTION_DESCRIPTION) _serviceaccount_help_text = CreateHelpText(_SERVICEACCOUNT_SYNOPSIS, _SERVICEACCOUNT_DESCRIPTION) _AUTHORIZE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), '-k': GcloudStorageFlag('--authorize-cmek'), }) _SERVICEACCOUNT_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), }) class KmsCommand(Command): """Implements of gsutil kms command.""" command_spec = Command.CreateCommandSpec( 'kms', usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='dk:p:w', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'authorize': [], 'encryption': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'serviceaccount': [], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='kms', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Cloud KMS encryption', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'authorize': _authorize_help_text, 'encryption': _encryption_help_text, 'serviceaccount': _serviceaccount_help_text }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'authorize': _AUTHORIZE_COMMAND, 'serviceaccount': _SERVICEACCOUNT_COMMAND, # "encryption" subcommand handled in get_gcloud_storage_args. }, flag_map={}) def get_gcloud_storage_args(self): if self.args[0] == 'encryption': gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'encryption': GcloudStorageMap( gcloud_command=['storage', 'buckets'], flag_map={ '-d': GcloudStorageFlag('--clear-default-encryption-key'), '-k': GcloudStorageFlag('--default-encryption-key'), '-w': GcloudStorageFlag(''), }), }, flag_map={}) if '-d' in self.args or '-k' in self.args: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'update' ] else: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'describe', ('--format=value[separator=\": \"](name, encryption' '.defaultKmsKeyName.yesno(no="No default encryption key."))'), '--raw' ] else: gcloud_storage_map = KmsCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GatherSubOptions(self, subcommand_name): self.CheckArguments() self.clear_kms_key = False self.kms_key = None self.warn_on_key_authorize_failure = False if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a elif o == '-k': self.kms_key = a ValidateCMEK(self.kms_key) elif o == '-d': self.clear_kms_key = True elif o == '-w': self.warn_on_key_authorize_failure = True if self.warn_on_key_authorize_failure and ( self.subcommand_name != 'encryption' or not self.kms_key): raise CommandException('\n'.join( textwrap.wrap( 'The "-w" option should only be specified for the "encryption" ' 'subcommand and must be used with the "-k" option.'))) # Determine the project (used in the serviceaccount and authorize # subcommands), either from the "-p" option's value or the default specified # in the user's Boto config file. if not self.project_id: self.project_id = PopulateProjectId(None) @Retry(ServiceException, tries=3, timeout_secs=1) def _AuthorizeProject(self, project_id, kms_key): """Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. """ # Request the Cloud Storage-owned service account for project_id, creating # it if it does not exist. service_account = self.gsutil_api.GetProjectServiceAccount( project_id, provider='gs').email_address kms_api = KmsApi(logger=self.logger) self.logger.debug('Getting IAM policy for %s', kms_key) try: policy = kms_api.GetKeyIamPolicy(kms_key) self.logger.debug('Current policy is %s', policy) # Check if the required binding is already present; if not, add it and # update the key's IAM policy. added_new_binding = False binding = Binding(role='roles/cloudkms.cryptoKeyEncrypterDecrypter', members=['serviceAccount:%s' % service_account]) if binding not in policy.bindings: policy.bindings.append(binding) kms_api.SetKeyIamPolicy(kms_key, policy) added_new_binding = True return (service_account, added_new_binding) except AccessDeniedException: if self.warn_on_key_authorize_failure: text_util.print_to_fd('\n'.join( textwrap.wrap( 'Warning: Check that your Cloud Platform project\'s service ' 'account has the "cloudkms.cryptoKeyEncrypterDecrypter" role ' 'for the specified key. Without this role, you may not be ' 'able to encrypt or decrypt objects using the key which will ' 'prevent you from uploading or downloading objects.'))) return (service_account, False) else: raise def _Authorize(self): self._GatherSubOptions('authorize') if not self.kms_key: raise CommandException('%s %s requires a key to be specified with -k' % (self.command_name, self.subcommand_name)) _, newly_authorized = self._AuthorizeProject(self.project_id, self.kms_key) if newly_authorized: print('Authorized project %s to encrypt and decrypt with key:\n%s' % (self.project_id, self.kms_key)) else: print('Project %s was already authorized to encrypt and decrypt with ' 'key:\n%s.' % (self.project_id, self.kms_key)) return 0 def _EncryptionClearKey(self, bucket_metadata, bucket_url): """Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. """ bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue() print('Clearing default encryption key for %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _EncryptionSetKey(self, bucket_metadata, bucket_url, svc_acct_for_project_num): """Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. """ bucket_project_number = bucket_metadata.projectNumber try: # newly_authorized will always be False if the project number is in our # cache dict, since we've already called _AuthorizeProject on it. service_account, newly_authorized = ( svc_acct_for_project_num[bucket_project_number], False) except KeyError: service_account, newly_authorized = self._AuthorizeProject( bucket_project_number, self.kms_key) svc_acct_for_project_num[bucket_project_number] = service_account if newly_authorized: text_util.print_to_fd('Authorized service account %s to use key:\n%s' % (service_account, self.kms_key)) bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue( defaultKmsKeyName=self.kms_key) print('Setting default KMS key for bucket %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _Encryption(self): self._GatherSubOptions('encryption') # For each project, we should only make one API call to look up its # associated Cloud Storage-owned service account; subsequent lookups can be # pulled from this cache dict. svc_acct_for_project_num = {} def _EncryptionForBucket(blr): """Set, clear, or get the defaultKmsKeyName for a bucket.""" bucket_url = blr.storage_url if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) # Determine the project from the provided bucket. bucket_metadata = self.gsutil_api.GetBucket( bucket_url.bucket_name, fields=['encryption', 'projectNumber'], provider=bucket_url.scheme) # "-d" flag was specified, so clear the default KMS key and return. if self.clear_kms_key: self._EncryptionClearKey(bucket_metadata, bucket_url) return 0 # "-k" flag was specified, so set the default KMS key and return. if self.kms_key: self._EncryptionSetKey(bucket_metadata, bucket_url, svc_acct_for_project_num) return 0 # Neither "-d" nor "-k" was specified, so emit the default KMS key and # return. bucket_url_string = str(bucket_url).rstrip('/') if (bucket_metadata.encryption and bucket_metadata.encryption.defaultKmsKeyName): print('Default encryption key for %s:\n%s' % (bucket_url_string, bucket_metadata.encryption.defaultKmsKeyName)) else: print('Bucket %s has no default encryption key' % bucket_url_string) return 0 # Iterate over bucket args, performing the specified encryption operation # for each. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _EncryptionForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _ServiceAccount(self): self.CheckArguments() if not self.args: self.args = ['gs://'] if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a if not self.project_id: self.project_id = PopulateProjectId(None) # Request the service account for that project; this might create the # service account if it doesn't already exist. self.logger.debug('Checking service account for project %s', self.project_id) service_account = self.gsutil_api.GetProjectServiceAccount( self.project_id, provider='gs').email_address print(service_account) return 0 def _RunSubCommand(self, func): try: self.sub_opts, self.args = getopt.getopt( self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API. If you ' 'have only supplied hmac credentials in your boto file, please ' 'instead supply a credential type that can be used with the JSON ' 'API.' % self.command_name))) def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API, which ' 'cannot use HMAC credentials. Please supply a credential ' 'type that is compatible with the JSON API (e.g. OAuth2) in your ' 'boto config file.' % self.command_name))) method_for_subcommand = { 'authorize': KmsCommand._Authorize, 'encryption': KmsCommand._Encryption, 'serviceaccount': KmsCommand._ServiceAccount } self.subcommand_name = self.args.pop(0) if self.subcommand_name in method_for_subcommand: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand(method_for_subcommand[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name)) hash.py 0000644 00000023504 15025166704 0006053 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of hash command for calculating hashes of local files.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import time import crcmod import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.progress_callback import FileProgressCallbackHandler from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.storage_url import StorageUrlFromString from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import hashing_helper from gslib.utils import parallelism_framework_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil hash [-c] [-h] [-m] filename... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Calculate hashes on local files, which can be used to compare with ``gsutil ls -L`` output. If a specific hash option is not provided, this command calculates all gsutil-supported hashes for the files. Note that gsutil automatically performs hash validation when uploading or downloading files, so this command is only needed if you want to write a script that separately checks the hash. If you calculate a CRC32c hash for files without a precompiled crcmod installation, hashing will be very slow. See "gsutil help crcmod" for details. <B>OPTIONS</B> -c Calculate a CRC32c hash for the specified files. -h Output hashes in hex format. By default, gsutil uses base64. -m Calculate a MD5 hash for the specified files. """) _GCLOUD_FORMAT_STRING = ( '--format=' + 'value[separator="",terminator=""](' + 'digest_format.sub("' + shim_util.get_format_flag_caret() + '", "Hashes ["),' + 'url.sub("' + shim_util.get_format_flag_caret() + '", "] for ").sub("$", ":^\\^n"),' + 'md5_hash.yesno(yes="\tHash (md5):\t\t", no=""),' + 'md5_hash.yesno(no=""),' + 'md5_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '"),' + 'crc32c_hash.yesno(yes="\tHash (crc32c):\t\t", no=""),' + 'crc32c_hash.yesno(no=""),' + 'crc32c_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '")' + ')') class HashCommand(Command): """Implementation of gsutil hash command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'hash', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='chm', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreFileURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='hash', help_name_aliases=['checksum'], help_type='command_help', help_one_line_summary='Calculate file hashes', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'hash', _GCLOUD_FORMAT_STRING, ], flag_map={ '-h': GcloudStorageFlag('--hex'), '-c': None, '-m': None, }, ) args_set = set(self.args + [flag for flag, _ in self.sub_opts]) if '-c' in args_set and '-m' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-md5'] elif '-m' in args_set and '-c' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-crc32c'] return super().get_gcloud_storage_args(gcloud_storage_map) @classmethod def _ParseOpts(cls, sub_opts, logger): """Returns behavior variables based on input options. Args: sub_opts: getopt sub-arguments for the command. logger: logging.Logger for the command. Returns: Tuple of calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum. calc_md5: Boolean, if True, command should calculate an MD5 hash. format_func: Function used for formatting the hash in the desired format. cloud_format_func: Function used for formatting the hash in the desired format. output_format: String describing the hash output format. """ calc_crc32c = False calc_md5 = False format_func = ( lambda digest: hashing_helper.Base64EncodeHash(digest.hexdigest())) cloud_format_func = lambda digest: digest found_hash_option = False output_format = 'base64' if sub_opts: for o, unused_a in sub_opts: if o == '-c': calc_crc32c = True found_hash_option = True elif o == '-h': output_format = 'hex' format_func = lambda digest: digest.hexdigest() cloud_format_func = lambda digest: ( hashing_helper.Base64ToHexHash(digest).decode('ascii') ) # yapf: disable elif o == '-m': calc_md5 = True found_hash_option = True if not found_hash_option: calc_crc32c = True calc_md5 = True if calc_crc32c and not boto_util.UsingCrcmodExtension(): logger.warn(hashing_helper.SLOW_CRCMOD_WARNING) return calc_crc32c, calc_md5, format_func, cloud_format_func, output_format def _GetHashClassesFromArgs(self, calc_crc32c, calc_md5): """Constructs the dictionary of hashes to compute based on the arguments. Args: calc_crc32c: If True, CRC32c should be included. calc_md5: If True, MD5 should be included. Returns: Dictionary of {string: hash digester}, where string the name of the digester algorithm. """ hash_dict = {} if calc_crc32c: hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') if calc_md5: hash_dict['md5'] = hashing_helper.GetMd5() return hash_dict def RunCommand(self): """Command entry point for the hash command.""" (calc_crc32c, calc_md5, format_func, cloud_format_func, output_format) = (self._ParseOpts(self.sub_opts, self.logger)) matched_one = False for url_str in self.args: for file_ref in self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=[ 'crc32c', 'customerEncryption', 'md5Hash', 'size', ]): matched_one = True url = StorageUrlFromString(url_str) file_name = file_ref.storage_url.object_name if StorageUrlFromString(url_str).IsFileUrl(): file_size = os.path.getsize(file_name) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=False, message_type=FileMessage.FILE_HASH)) callback_processor = ProgressCallbackWithTimeout( file_size, FileProgressCallbackHandler(self.gsutil_api.status_queue, src_url=StorageUrlFromString(url_str), operation_name='Hashing').call) hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5) with open(file_name, 'rb') as fp: hashing_helper.CalculateHashesFromContents( fp, hash_dict, callback_processor=callback_processor) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=True, message_type=FileMessage.FILE_HASH)) else: hash_dict = {} obj_metadata = file_ref.root_object file_size = obj_metadata.size md5_present = obj_metadata.md5Hash is not None crc32c_present = obj_metadata.crc32c is not None if not md5_present and not crc32c_present: logging.getLogger().warn('No hashes present for %s', url_str) continue if md5_present: hash_dict['md5'] = obj_metadata.md5Hash if crc32c_present: hash_dict['crc32c'] = obj_metadata.crc32c print('Hashes [%s] for %s:' % (output_format, file_name)) for name, digest in six.iteritems(hash_dict): print('\tHash (%s):\t\t%s' % (name, (format_func(digest) if url.IsFileUrl() else cloud_format_func(digest)))) if not matched_one: raise CommandException('No files matched') _PutToQueueWithTimeout(self.gsutil_api.status_queue, FinalMessage(time.time())) return 0 iam.py 0000644 00000104005 15025166704 0005672 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of IAM policy management command for GCS.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import json import os import re import subprocess import textwrap import six from six.moves import zip from apitools.base.protorpclite import protojson from apitools.base.protorpclite.messages import DecodeError from boto import config from gslib.cloud_api import ArgumentException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import GetFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import IamChOnResourceWithConditionsException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.storage_url import GetSchemeFromUrlString from gslib.storage_url import IsKnownUrlScheme from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import IAM_POLICY_VERSION from gslib.utils.constants import NO_MAX from gslib.utils import iam_helper from gslib.utils.iam_helper import BindingStringToTuple from gslib.utils.iam_helper import BindingsTuple from gslib.utils.iam_helper import DeserializeBindingsTuple from gslib.utils.iam_helper import IsEqualBindings from gslib.utils.iam_helper import PatchBindings from gslib.utils.iam_helper import SerializeBindingsTuple from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.shim_util import GcloudStorageFlag _SET_SYNOPSIS = """ gsutil iam set [-afRr] [-e <etag>] file url ... """ _GET_SYNOPSIS = """ gsutil iam get url """ # Note that the commands below are put in quotation marks instead of backticks; # this is done because the whole ch synopsis gets rendered in one <pre> tag in # the web docs, and having literal double-backticks looks weird. _CH_SYNOPSIS = """ gsutil iam ch [-fRr] binding ... url where each binding is of the form: [-d] ("user"|"serviceAccount"|"domain"|"group"):id:role[,...] [-d] ("allUsers"|"allAuthenticatedUsers"):role[,...] -d ("user"|"serviceAccount"|"domain"|"group"):id -d ("allUsers"|"allAuthenticatedUsers") NOTE: The "iam ch" command does not support changing Cloud IAM policies with bindings that contain conditions. As such, "iam ch" cannot be used to add conditions to a policy or to change the policy of a resource that already contains conditions. See additional details below. NOTE: The "gsutil iam" command does not allow you to add convenience values (projectOwner, projectEditor, projectViewer), but you can remove existing ones. """ _GET_DESCRIPTION = """ <B>GET</B> The ``iam get`` command gets the Cloud IAM policy for a bucket or object, which you can save and edit for use with the ``iam set`` command. The following examples save the bucket or object's Cloud IAM policy to a text file: gsutil iam get gs://example > bucket_iam.txt gsutil iam get gs://example/important.txt > object_iam.txt The Cloud IAM policy returned by ``iam get`` includes an etag. The etag is used in the precondition check for ``iam set`` unless you override it using ``iam set -e``. """ _SET_DESCRIPTION = """ <B>SET</B> The ``iam set`` command sets a Cloud IAM policy on one or more buckets or objects, replacing the existing policy on those buckets or objects. For an example of the correct formatting for a Cloud IAM policy, see the output of the ``iam get`` command. You can use the ``iam ch`` command to edit an existing policy, even in the presence of concurrent updates. You can also edit the policy concurrently using the ``-e`` flag to override the Cloud IAM policy's etag. Specifying ``-e`` with an empty string (i.e. ``gsutil iam set -e '' ...``) instructs gsutil to skip the precondition check when setting the Cloud IAM policy. When you set a Cloud IAM policy on a large number of objects, you should use the gsutil ``-m`` option for concurrent processing. The following command applies ``iam.txt`` to all objects in the ``dogs`` bucket: gsutil -m iam set -r iam.txt gs://dogs Note that only object-level operations are parallelized; setting a Cloud IAM policy on a large number of buckets with the ``-m`` flag does not improve performance. <B>SET OPTIONS</B> The ``set`` sub-command has the following options: -R, -r Performs ``iam set`` recursively on all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -a Performs ``iam set`` on all object versions. -e <etag> Performs the precondition check on each object with the specified etag before setting the policy. You can retrieve the policy's etag using ``iam get``. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This option is implicitly set when you use the gsutil ``-m`` option. """ _CH_DESCRIPTION = """ <B>CH</B> The ``iam ch`` command incrementally updates Cloud IAM policies. You can specify multiple access grants or removals in a single command. The access changes are applied as a batch to each url in the order in which they appear in the command line arguments. Each access change specifies a principal and a role that is either granted or revoked. You can use gsutil ``-m`` to handle object-level operations in parallel. NOTE: The ``iam ch`` command cannot be used to change the Cloud IAM policy of a resource that contains conditions in its policy bindings. Attempts to do so result in an error. To change the Cloud IAM policy of such a resource, you can perform a read-modify-write operation by saving the policy to a file using ``iam get``, editing the file, and setting the updated policy using ``iam set``. <B>CH EXAMPLES</B> Examples for the ``ch`` sub-command: To grant a single role to a single principal for some targets: gsutil iam ch user:john.doe@example.com:objectCreator gs://ex-bucket To make a bucket's objects publicly readable: gsutil iam ch allUsers:objectViewer gs://ex-bucket To grant multiple bindings to a bucket: gsutil iam ch user:john.doe@example.com:objectCreator \\ domain:www.my-domain.org:objectViewer gs://ex-bucket To specify more than one role for a particular principal: gsutil iam ch user:john.doe@example.com:objectCreator,objectViewer \\ gs://ex-bucket To specify a custom role for a particular principal: gsutil iam ch user:john.doe@example.com:roles/customRoleName gs://ex-bucket To apply a grant and simultaneously remove a binding to a bucket: gsutil iam ch -d group:readers@example.com:legacyBucketReader \\ group:viewers@example.com:objectViewer gs://ex-bucket To remove a user from all roles on a bucket: gsutil iam ch -d user:john.doe@example.com gs://ex-bucket <B>CH OPTIONS</B> The ``ch`` sub-command has the following options: -d Removes roles granted to the specified principal. -R, -r Performs ``iam ch`` recursively to all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This is implicitly set when you invoke the gsutil ``-m`` option. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Cloud Identity and Access Management (Cloud IAM) allows you to control who has access to the resources in your Google Cloud project. For more information, see `Cloud Identity and Access Management <https://cloud.google.com/storage/docs/access-control/iam>`_. The iam command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) STORAGE_URI_REGEX = re.compile(r'[a-z]+://.+') IAM_CH_CONDITIONS_WORKAROUND_MSG = ( 'To change the IAM policy of a resource that has bindings containing ' 'conditions, perform a read-modify-write operation using "iam get" and ' '"iam set".') def _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(urls, recursion_requested): if UrlsAreMixOfBucketsAndObjects(urls) and not recursion_requested: raise CommandException('Cannot operate on a mix of buckets and objects.') def _PatchIamWrapper(cls, iter_result, thread_state): (serialized_bindings_tuples, expansion_result) = iter_result return cls.PatchIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. [DeserializeBindingsTuple(t) for t in serialized_bindings_tuples], thread_state=thread_state) def _SetIamWrapper(cls, iter_result, thread_state): (serialized_policy, expansion_result) = iter_result return cls.SetIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. protojson.decode_message(apitools_messages.Policy, serialized_policy), thread_state=thread_state) def _SetIamExceptionHandler(cls, e): cls.logger.error(str(e)) def _PatchIamExceptionHandler(cls, e): cls.logger.error(str(e)) class IamCommand(Command): """Implementation of gsutil iam command.""" command_spec = Command.CreateCommandSpec( 'iam', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrd:e:', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'ch': [ CommandArgument.MakeOneOrMoreBindingsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], }, ) help_spec = Command.HelpSpec( help_name='iam', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change bucket and/or object IAM permissions.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def _get_resource_type(self, url_patterns): if self.recursion_requested or url_patterns[0].IsObject(): return 'objects' else: return 'buckets' def get_gcloud_storage_args(self): self.sub_command = self.args.pop(0) if self.sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'get-iam-policy', '--format=json' ], flag_map={}) elif self.sub_command == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', None, 'set-iam-policy', '--format=json'], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-e': GcloudStorageFlag('--etag'), # Hack to make gcloud recognize empty string value. '_empty_etag': GcloudStorageFlag('--etag='), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() # Flags moved to self.sub_opts, leaving [POLICY-FILE, URLS...]. url_strings = self.args[1:] url_objects = list(map(StorageUrlFromString, url_strings)) recurse = False for i, (flag_key, flag_value) in enumerate(self.sub_opts): if flag_key in ('-r', '-R'): recurse = True elif flag_key == '-e' and flag_value == '': # Allow for empty string values. self.sub_opts[i] = ('_empty_etag', '') _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, recurse) if recurse or url_objects[0].IsObject(): gcloud_storage_map.gcloud_command[1] = 'objects' else: gcloud_storage_map.gcloud_command[1] = 'buckets' # Gsutil takes policy file then URLs, and gcloud storage does the reverse. self.args = url_strings + self.args[:1] elif self.sub_command == 'ch': if shim_util.HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) is shim_util.HIDDEN_SHIM_MODE.DRY_RUN: self.logger.warning( 'The shim maps iam ch commands to several gcloud storage commands,' ' which cannot be determined without running gcloud storage.') return [] # Translation occurs in self.run_gcloud_storage. return super().get_gcloud_storage_args(gcloud_storage_map) def _RaiseIfInvalidUrl(self, url): if url.IsFileUrl(): error_msg = 'Invalid Cloud URL "%s".' % url.object_name if set(url.object_name).issubset(set('-Rrf')): error_msg += ( ' This resource handle looks like a flag, which must appear' ' before all bindings. See "gsutil help iam ch" for more details.') raise CommandException(error_msg) def _GetSettingsAndDiffs(self): self.continue_on_error = False self.recursion_requested = False patch_bindings_tuples = [] if self.sub_opts: for o, a in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-d': patch_bindings_tuples.append(BindingStringToTuple(False, a)) url_pattern_strings = [] # N.B.: self.sub_opts stops taking in options at the first non-flagged # token. The rest of the tokens are sent to self.args. Thus, in order to # handle input of the form "-d <binding> <binding> <url>", we will have to # parse self.args for a mix of both bindings and CloudUrls. We are not # expecting to come across the -r, -f flags here. it = iter(self.args) for token in it: if (STORAGE_URI_REGEX.match(token) and IsKnownUrlScheme(GetSchemeFromUrlString(token))): url_pattern_strings.append(token) break if token == '-d': try: patch_bindings_tuples.append(BindingStringToTuple(False, next(it))) except StopIteration: raise CommandException( 'A -d flag is missing an argument specifying bindings to remove.') else: patch_bindings_tuples.append(BindingStringToTuple(True, token)) if not patch_bindings_tuples: raise CommandException('Must specify at least one binding.') # All following arguments are urls. for token in it: url_pattern_strings.append(token) url_objects = list(map(StorageUrlFromString, url_pattern_strings)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, self.recursion_requested) return patch_bindings_tuples, url_objects def _run_ch_subprocess(self, args, stdin=None): _, command = self._get_full_gcloud_storage_execution_information(args) process = subprocess.run( command, env=self._get_shim_command_environment_variables(), input=stdin, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True, ) if process.returncode != 0: if self.continue_on_error: self.logger.error(process.stderr) else: raise CommandException(process.stderr) return process def run_gcloud_storage(self): if self.sub_command != 'ch': return super().run_gcloud_storage() self.ParseSubOpts() bindings_tuples, patterns = self._GetSettingsAndDiffs() resource_type = self._get_resource_type(patterns) list_settings = [] if self.recursion_requested: list_settings.append('-r') return_code = 0 for url_pattern in patterns: self._RaiseIfInvalidUrl(url_pattern) if resource_type == 'objects': ls_process = self._run_ch_subprocess(['storage', 'ls', '--json'] + list_settings + [str(url_pattern)]) if ls_process.returncode: return_code = 1 continue ls_output = json.loads(ls_process.stdout) urls = [ resource['url'] for resource in ls_output if resource['type'] == 'cloud_object' ] else: urls = [str(url_pattern)] for url in urls: get_process = self._run_ch_subprocess( ['storage', resource_type, 'get-iam-policy', url, '--format=json']) if get_process.returncode: return_code = 1 continue policy = json.loads(get_process.stdout) bindings = iam_helper.BindingsDictToUpdateDict(policy['bindings']) for (is_grant, diff) in bindings_tuples: diff_dict = iam_helper.BindingsDictToUpdateDict(diff) bindings = PatchBindings(bindings, diff_dict, is_grant) policy['bindings'] = [{ 'role': r, 'members': sorted(list(m)) } for r, m in sorted(six.iteritems(bindings))] set_process = self._run_ch_subprocess( ['storage', resource_type, 'set-iam-policy', url, '-'], stdin=json.dumps(policy, sort_keys=True)) if set_process.returncode: return_code = 1 return return_code def GetIamHelper(self, storage_url, thread_state=None): """Gets an IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the global -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Returns: Policy instance. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): policy = gsutil_api.GetBucketIamPolicy( storage_url.bucket_name, provider=storage_url.scheme, fields=['bindings', 'etag'], ) else: policy = gsutil_api.GetObjectIamPolicy( storage_url.bucket_name, storage_url.object_name, generation=storage_url.generation, provider=storage_url.scheme, fields=['bindings', 'etag'], ) return policy def _GetIam(self, thread_state=None): """Gets IAM policy for single bucket or object.""" pattern = self.args[0] matches = PluralityCheckableIterator( self.WildcardIterator(pattern).IterAll(bucket_listing_fields=['name'])) if matches.IsEmpty(): raise CommandException('%s matched no URLs' % pattern) if matches.HasPlurality(): raise CommandException( '%s matched more than one URL, which is not allowed by the %s ' 'command' % (pattern, self.command_name)) storage_url = StorageUrlFromString(list(matches)[0].url_string) policy = self.GetIamHelper(storage_url, thread_state=thread_state) policy_json = json.loads(protojson.encode_message(policy)) policy_str = json.dumps( policy_json, sort_keys=True, separators=(',', ': '), indent=2, ) print(policy_str) def _SetIamHelperInternal(self, storage_url, policy, thread_state=None): """Sets IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. policy: A Policy object to set on the bucket / object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Raises: ServiceException passed from the API call if an HTTP error was returned. """ # SetIamHelper may be called by a command.WorkerThread. In the # single-threaded case, WorkerThread will not pass the CloudApiDelegator # instance to thread_state. GetCloudInstance is called to resolve the # edge case. gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): gsutil_api.SetBucketIamPolicy(storage_url.bucket_name, policy, provider=storage_url.scheme) else: gsutil_api.SetObjectIamPolicy(storage_url.bucket_name, storage_url.object_name, policy, generation=storage_url.generation, provider=storage_url.scheme) def SetIamHelper(self, storage_url, policy, thread_state=None): """Handles the potential exception raised by the internal set function.""" try: self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise def PatchIamHelper(self, storage_url, bindings_tuples, thread_state=None): """Patches an IAM policy for a single, resolved bucket / object URL. The patch is applied by altering the policy from an IAM get request, and setting the new IAM with the specified etag. Because concurrent IAM set requests may alter the etag, we may need to retry this operation several times before success. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. bindings_tuples: A list of BindingsTuple instances. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. """ try: self._PatchIamHelperInternal(storage_url, bindings_tuples, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise except IamChOnResourceWithConditionsException as e: if self.continue_on_error: self.everything_set_okay = False self.tried_ch_on_resource_with_conditions = True self.logger.debug(e.message) else: raise CommandException(e.message) @Retry(PreconditionException, tries=3, timeout_secs=1.0) def _PatchIamHelperInternal(self, storage_url, bindings_tuples, thread_state=None): policy = self.GetIamHelper(storage_url, thread_state=thread_state) (etag, bindings) = (policy.etag, policy.bindings) # If any of the bindings have conditions present, raise an exception. # See the docstring for the IamChOnResourceWithConditionsException class # for more details on why we raise this exception. for binding in bindings: if binding.condition: message = 'Could not patch IAM policy for %s.' % storage_url message += '\n' message += '\n'.join( textwrap.wrap( 'The resource had conditions present in its IAM policy bindings, ' 'which is not supported by "iam ch". %s' % IAM_CH_CONDITIONS_WORKAROUND_MSG)) raise IamChOnResourceWithConditionsException(message) # Create a backup which is untainted by any references to the original # bindings. orig_bindings = list(bindings) for (is_grant, diff) in bindings_tuples: bindings_dict = iam_helper.BindingsMessageToUpdateDict(bindings) diff_dict = iam_helper.BindingsMessageToUpdateDict(diff) new_bindings_dict = PatchBindings(bindings_dict, diff_dict, is_grant) bindings = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(new_bindings_dict) ] if IsEqualBindings(bindings, orig_bindings): self.logger.info('No changes made to %s', storage_url) return policy = apitools_messages.Policy(bindings=bindings, etag=etag) # We explicitly wish for etag mismatches to raise an error and allow this # function to error out, so we are bypassing the exception handling offered # by IamCommand.SetIamHelper in lieu of our own handling (@Retry). self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) def _PatchIam(self): raw_bindings_tuples, url_patterns = self._GetSettingsAndDiffs() patch_bindings_tuples = [] for is_grant, bindings in raw_bindings_tuples: bindings_messages = [] for binding in bindings: bindings_message = apitools_messages.Policy.BindingsValueListEntry( members=binding['members'], role=binding['role']) bindings_messages.append(bindings_message) patch_bindings_tuples.append( BindingsTuple(is_grant=is_grant, bindings=bindings_messages)) self.everything_set_okay = True self.tried_ch_on_resource_with_conditions = False threaded_wildcards = [] for surl in url_patterns: try: if surl.IsBucket(): if self.recursion_requested: surl.object = '*' threaded_wildcards.append(surl.url_string) else: self.PatchIamHelper(surl, patch_bindings_tuples) else: threaded_wildcards.append(surl.url_string) except AttributeError: self._RaiseIfInvalidUrl(surl) if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) serialized_bindings_tuples_it = itertools.repeat( [SerializeBindingsTuple(t) for t in patch_bindings_tuples]) self.Apply(_PatchIamWrapper, zip(serialized_bindings_tuples_it, name_expansion_iterator), _PatchIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: msg = 'Some IAM policies could not be patched.' if self.tried_ch_on_resource_with_conditions: msg += '\n' msg += '\n'.join( textwrap.wrap( 'Some resources had conditions present in their IAM policy ' 'bindings, which is not supported by "iam ch". %s' % (IAM_CH_CONDITIONS_WORKAROUND_MSG))) raise CommandException(msg) # TODO(iam-beta): Add an optional flag to specify etag and edit the policy # accordingly to be passed into the helper functions. def _SetIam(self): """Set IAM policy for given wildcards on the command line.""" self.continue_on_error = False self.recursion_requested = False self.all_versions = False force_etag = False etag = '' if self.sub_opts: for o, arg in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-a': self.all_versions = True elif o == '-e': etag = str(arg) force_etag = True else: self.RaiseInvalidArgumentException() file_url = self.args[0] patterns = self.args[1:] # Load the IAM policy file and raise error if the file is invalid JSON or # does not exist. try: with open(file_url, 'r') as fp: policy = json.loads(fp.read()) except IOError: raise ArgumentException('Specified IAM policy file "%s" does not exist.' % file_url) except ValueError as e: self.logger.debug('Invalid IAM policy file, ValueError:\n%s', e) raise ArgumentException('Invalid IAM policy file "%s".' % file_url) bindings = policy.get('bindings', []) if not force_etag: etag = policy.get('etag', '') policy_json = json.dumps({ 'bindings': bindings, 'etag': etag, 'version': IAM_POLICY_VERSION }) try: policy = protojson.decode_message(apitools_messages.Policy, policy_json) except DecodeError: raise ArgumentException('Invalid IAM policy file "%s" or etag "%s".' % (file_url, etag)) self.everything_set_okay = True # This list of wildcard strings will be handled by NameExpansionIterator. threaded_wildcards = [] surls = list(map(StorageUrlFromString, patterns)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(surls, self.recursion_requested) for surl in surls: print(surl.url_string) if surl.IsBucket(): if self.recursion_requested: surl.object_name = '*' threaded_wildcards.append(surl.url_string) else: self.SetIamHelper(surl, policy) else: threaded_wildcards.append(surl.url_string) # N.B.: If threaded_wildcards contains a non-existent bucket # (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator # will raise an exception in iter.next. This halts all iteration, even # when -f is set. This behavior is also evident in acl set. This behavior # also appears for any exception that will be raised when iterating over # wildcard expansions (access denied if bucket cannot be listed, etc.). if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) policy_it = itertools.repeat(protojson.encode_message(policy)) self.Apply(_SetIamWrapper, zip(policy_it, name_expansion_iterator), _SetIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: raise CommandException('Some IAM policies could not be set.') def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': LogCommandParams(subcommands=[action_subcommand]) self._GetIam() elif action_subcommand == 'set': LogCommandParams(subcommands=[action_subcommand]) self._SetIam() elif action_subcommand == 'ch': LogCommandParams(subcommands=[action_subcommand]) self._PatchIam() else: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help iam".' % (action_subcommand, self.command_name)) return 0 help.py 0000644 00000022271 15025166704 0006060 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil help command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import os import pkgutil import re import six from subprocess import PIPE from subprocess import Popen import gslib.addlhelp from gslib.command import Command from gslib.command import OLD_ALIAS_MAP import gslib.commands from gslib.exception import CommandException from gslib.help_provider import HelpProvider from gslib.help_provider import MAX_HELP_NAME_LEN from gslib.utils import constants from gslib.utils.system_util import IS_WINDOWS from gslib.utils.system_util import IsRunningInteractively from gslib.utils.system_util import GetTermLines from gslib.utils import text_util _SYNOPSIS = """ gsutil help [command or topic] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The following command provides a summary of gsutil commands and additional topics on which help is available: gsutil help The following command provides help about the specified command or topic: gsutil help [command or topic] The following command provides help about the specified sub-command: gsutil help [command] [sub-command] For example, running the following provides help about the "set" sub-command of the "acl" command: gsutil help acl set If you set the PAGER environment variable to the path to a pager program (such as /bin/less on Linux), long help sections are piped through the specified pager. """) top_level_usage_string = ( 'Usage: gsutil [-D] [-DD] [-h header]... [-i service_account] ' '[-m] [-o section:flag=value]... [-q] [-u user_project] [command [opts...] args...]' ) class HelpCommand(Command): """Implementation of gsutil help command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'help', command_name_aliases=['?', 'man'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=2, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='help', help_name_aliases=['?'], help_type='command_help', help_one_line_summary='Get help about commands and topics', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the help command.""" (help_type_map, help_name_map) = self._LoadHelpMaps() output = [] if not self.args: output.append('%s\nAvailable commands:\n' % top_level_usage_string) format_str = ' %-' + str(MAX_HELP_NAME_LEN) + 's%s\n' for help_prov in sorted(help_type_map['command_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nAdditional help topics:\n') for help_prov in sorted(help_type_map['additional_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nUse gsutil help <command or topic> for detailed help.') else: invalid_subcommand = False arg = self.args[0] if arg not in help_name_map: output.append('No help available for "%s"' % arg) else: help_prov = help_name_map[arg] help_name = None if len(self.args) > 1: # We also have a subcommand argument. subcommand_map = help_prov.help_spec.subcommand_help_text if subcommand_map and self.args[1] in subcommand_map: help_name = arg + ' ' + self.args[1] help_text = subcommand_map[self.args[1]] else: invalid_subcommand = True if not subcommand_map: output.append( ('The "%s" command has no subcommands. You can ask for the ' 'full help by running:\n\n\tgsutil help %s\n') % (arg, arg)) else: subcommand_examples = [] for subcommand in subcommand_map: subcommand_examples.append('\tgsutil help %s %s' % (arg, subcommand)) output.append( ('Subcommand "%s" does not exist for command "%s".\n' 'You can either ask for the full help about the command by ' 'running:\n\n\tgsutil help %s\n\n' 'Or you can ask for help about one of the subcommands:\n\n%s' ) % (self.args[1], arg, arg, '\n'.join(subcommand_examples))) if not invalid_subcommand: if not help_name: # No subcommand or invalid subcommand. help_name = help_prov.help_spec.help_name help_text = help_prov.help_spec.help_text output.append('<B>NAME</B>\n') output.append(' %s - %s\n' % (help_name, help_prov.help_spec.help_one_line_summary)) output.append('\n\n') output.append(help_text.strip('\n')) new_alias = OLD_ALIAS_MAP.get(arg, [None])[0] if new_alias: deprecation_warning = """ The "%s" alias is deprecated, and will eventually be removed completely. Please use the "%s" command instead.""" % (arg, new_alias) output.append('\n\n\n<B>DEPRECATION WARNING</B>\n') output.append(deprecation_warning) self._OutputHelp(''.join(output)) return 0 def _OutputHelp(self, help_str): """Outputs simply formatted string. This function paginates if the string is too long, PAGER is defined, and the output is a tty. Args: help_str: String to format. """ # Remove <B> and </B> tags and replace them with ANSI control codes if # writing to a compatible tty. if IS_WINDOWS or not IsRunningInteractively(): help_str = re.sub('<B>', '', help_str) help_str = re.sub('</B>', '', help_str) text_util.print_to_fd(help_str) return help_str = re.sub('<B>', '\033[1m', help_str) help_str = re.sub('</B>', '\033[0;0m', help_str) num_lines = len(help_str.split('\n')) if 'PAGER' in os.environ and num_lines >= GetTermLines(): # Use -r option for less to make bolding work right. pager = os.environ['PAGER'].split(' ') if pager[0].endswith('less'): pager.append('-r') try: if six.PY2: input_for_pager = help_str.encode(constants.UTF8) else: input_for_pager = help_str Popen(pager, stdin=PIPE, universal_newlines=True).communicate(input=input_for_pager) except OSError as e: raise CommandException('Unable to open pager (%s): %s' % (' '.join(pager), e)) else: text_util.print_to_fd(help_str) def _LoadHelpMaps(self): """Returns tuple of help type and help name. help type is a dict with key: help type value: list of HelpProviders help name is a dict with key: help command name or alias value: HelpProvider Returns: (help type, help name) """ # Import all gslib.commands submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): __import__('gslib.commands.%s' % module_name) # Import all gslib.addlhelp submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.addlhelp.__path__): __import__('gslib.addlhelp.%s' % module_name) help_type_map = {} help_name_map = {} for s in gslib.help_provider.ALL_HELP_TYPES: help_type_map[s] = [] # Only include HelpProvider subclasses in the dict. for help_prov in itertools.chain(HelpProvider.__subclasses__(), Command.__subclasses__()): if help_prov is Command: # Skip the Command base class itself; we just want its subclasses, # where the help command text lives (in addition to non-Command # HelpProviders, like naming.py). continue gslib.help_provider.SanityCheck(help_prov, help_name_map) help_name_map[help_prov.help_spec.help_name] = help_prov for help_name_aliases in help_prov.help_spec.help_name_aliases: help_name_map[help_name_aliases] = help_prov help_type_map[help_prov.help_spec.help_type].append(help_prov) return (help_type_map, help_name_map) setmeta.py 0000644 00000037563 15025166704 0006604 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of setmeta command for setting cloud object metadata.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from apitools.base.py import encoding from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.metadata_util import IsCustomMetadataHeader from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAsciiHeader from gslib.utils.text_util import InsistAsciiHeaderValue from gslib.utils.translation_helper import CopyObjectMetadata from gslib.utils.translation_helper import ObjectMetadataFromHeaders from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil setmeta -h [header:value|header] ... url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil setmeta command allows you to set or remove the metadata on one or more objects. It takes one or more header arguments followed by one or more URLs, where each header argument is in one of two forms: - If you specify ``header:value``, it sets the provided value for the given header on all applicable objects. - If you specify ``header`` (with no value), it removes the given header from all applicable objects. For example, the following command sets the ``Content-Type`` and ``Cache-Control`` headers while also removing the ``Content-Disposition`` header on the specified objects: gsutil setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/*.html If you have a large number of objects to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/*.html You can also use the setmeta command to set custom metadata on an object: gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object Custom metadata is always prefixed in gsutil with ``x-goog-meta-``. This distinguishes it from standard request headers. Other tools that send and receive object metadata by using the request body do not use this prefix. While gsutil supports custom metadata with arbitrary Unicode values, note that when setting metadata using the XML API, which sends metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. The setmeta command reads each object's current generation and metageneration and uses those as preconditions unless they are otherwise specified by top-level arguments. For example, the following command sets the custom metadata ``icecreamflavor:vanilla`` if the current live object has a metageneration of 2: gsutil -h "x-goog-if-metageneration-match:2" setmeta -h "x-goog-meta-icecreamflavor:vanilla" See `Object metadata <https://cloud.google.com/storage/docs/metadata>`_ for more information about object metadata. <B>OPTIONS</B> -h Specifies a header:value to be added, or header to be removed, from each named object. """) # Setmeta assumes a header-like model which doesn't line up with the JSON way # of doing things. This list comes from functionality that was supported by # gsutil3 at the time gsutil4 was released. SETTABLE_FIELDS = [ 'cache-control', 'content-disposition', 'content-encoding', 'content-language', 'content-type', 'custom-time' ] _GCLOUD_OBJECTS_UPDATE_COMMAND = ['storage', 'objects', 'update'] def _SetMetadataExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(e) cls.everything_set_okay = False def _SetMetadataFuncWrapper(cls, name_expansion_result, thread_state=None): cls.SetMetadataFunc(name_expansion_result, thread_state=thread_state) class SetMetaCommand(Command): """Implementation of gsutil setmeta command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'setmeta', command_name_aliases=['setheader'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='h:rR', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='setmeta', help_name_aliases=['setheader'], help_type='command_help', help_one_line_summary='Set metadata on already uploaded objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=_GCLOUD_OBJECTS_UPDATE_COMMAND, flag_map={}, ) def get_gcloud_storage_args(self): recursive_flag = [] for o, _ in self.sub_opts: if o in ('-r', '-R'): recursive_flag = ['--recursive'] break clear_set, set_dict = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) # Handle translation through "gcloud_command" instead of "flag_map". self.sub_opts = [] clear_flags = self._translate_headers( {clear_key: None for clear_key in clear_set}, unset=True) set_flags = self._translate_headers(set_dict, unset=False) command = (_GCLOUD_OBJECTS_UPDATE_COMMAND + recursive_flag + clear_flags + set_flags) gcloud_storage_map = GcloudStorageMap( gcloud_command=command, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the setmeta command.""" metadata_minus, metadata_plus = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) self.metadata_change = metadata_plus for header in metadata_minus: self.metadata_change[header] = '' if not self.metadata_change: raise CommandException( 'gsutil setmeta requires one or more headers to be provided with the' ' -h flag. See "gsutil help setmeta" for more information.') if len(self.args) == 1 and not self.recursion_requested: url = StorageUrlFromString(self.args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL (%s) must name an object' % self.args[0]) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True self.preconditions = PreconditionsFromHeaders(self.headers) name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, self.args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metadata', 'metageneration']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) try: # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_SetMetadataFuncWrapper, name_expansion_iterator, _SetMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('Metadata for some objects could not be set.') return 0 @Retry(PreconditionException, tries=3, timeout_secs=1) def SetMetadataFunc(self, name_expansion_result, thread_state=None): """Sets metadata on an object. Args: name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Setting metadata on %s...', exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration # Patch handles the patch semantics for most metadata, but we need to # merge the custom metadata field manually. patch_obj_metadata = ObjectMetadataFromHeaders(self.metadata_change) api = gsutil_api.GetApiSelector(provider=exp_src_url.scheme) # For XML we only want to patch through custom metadata that has # changed. For JSON we need to build the complete set. if api == ApiSelector.XML: pass elif api == ApiSelector.JSON: CopyObjectMetadata(patch_obj_metadata, cloud_obj_metadata, override=True) patch_obj_metadata = cloud_obj_metadata # Patch body does not need the object generation and metageneration. patch_obj_metadata.generation = None patch_obj_metadata.metageneration = None gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetHeaderStringsFromSubOpts(self): """Gets header values from after the "setmeta" part of the command. Example: $ gsutil -h not:parsed setmeta is:parsed gs://bucket/object -> ["is:parsed"] Returns: List[str]: Headers without the "-h" but not yet split on colons. Raises: CommandException Found canned ACL. """ if not self.sub_opts: return [] headers = [] for o, a in self.sub_opts: if o == '-h': if 'x-goog-acl' in a or 'x-amz-acl' in a: raise CommandException( 'gsutil setmeta no longer allows canned ACLs. Use gsutil acl ' 'set ... to set canned ACLs.') headers.append(a) return headers def _ParseMetadataHeaders(self, headers): """Validates and parses metadata changes from the headers argument. Args: headers: Header dict to validate and parse. Returns: (metadata_plus, metadata_minus): Tuple of header sets to add and remove. """ metadata_minus = set() cust_metadata_minus = set() metadata_plus = {} cust_metadata_plus = {} # Build a count of the keys encountered from each plus and minus arg so we # can check for dupe field specs. num_metadata_plus_elems = 0 num_cust_metadata_plus_elems = 0 num_metadata_minus_elems = 0 num_cust_metadata_minus_elems = 0 for md_arg in headers: # Use partition rather than split, as we should treat all characters past # the initial : as part of the header's value. parts = md_arg.partition(':') (header, _, value) = parts InsistAsciiHeader(header) # Translate headers to lowercase to match the casing assumed by our # sanity-checking operations. lowercase_header = header.lower() # This check is overly simple; it would be stronger to check, for each # URL argument, whether the header starts with the provider # metadata_prefix, but here we just parse the spec once, before # processing any of the URLs. This means we will not detect if the user # tries to set an x-goog-meta- field on an another provider's object, # for example. is_custom_meta = IsCustomMetadataHeader(lowercase_header) if not is_custom_meta and lowercase_header not in SETTABLE_FIELDS: raise CommandException( 'Invalid or disallowed header (%s).\nOnly these fields (plus ' 'x-goog-meta-* fields) can be set or unset:\n%s' % (header, sorted(list(SETTABLE_FIELDS)))) if value: if is_custom_meta: # Allow non-ASCII data for custom metadata fields. cust_metadata_plus[header] = value num_cust_metadata_plus_elems += 1 else: # Don't unicode encode other fields because that would perturb their # content (e.g., adding %2F's into the middle of a Cache-Control # value). InsistAsciiHeaderValue(header, value) value = str(value) metadata_plus[lowercase_header] = value num_metadata_plus_elems += 1 else: if is_custom_meta: cust_metadata_minus.add(header) num_cust_metadata_minus_elems += 1 else: metadata_minus.add(lowercase_header) num_metadata_minus_elems += 1 if (num_metadata_plus_elems != len(metadata_plus) or num_cust_metadata_plus_elems != len(cust_metadata_plus) or num_metadata_minus_elems != len(metadata_minus) or num_cust_metadata_minus_elems != len(cust_metadata_minus) or metadata_minus.intersection(set(metadata_plus.keys()))): raise CommandException('Each header must appear at most once.') metadata_plus.update(cust_metadata_plus) metadata_minus.update(cust_metadata_minus) return (metadata_minus, metadata_plus) mv.py 0000644 00000014555 15025166704 0005560 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like mv command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.cp import CP_AND_MV_SHIM_FLAG_MAP from gslib.commands.cp import CP_SUB_ARGS from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil mv [-p] src_url dst_url gsutil mv [-p] src_url... dst_url gsutil mv [-p] -I dst_url """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil mv`` command allows you to move data between your local file system and the cloud, move data within the cloud, and move data between cloud storage providers. For example, to move all objects from a bucket to a local directory you could use: gsutil mv gs://my_bucket/* dir Similarly, to move all objects from a local directory to a bucket you could use: gsutil mv ./dir gs://my_bucket <B>RENAMING GROUPS OF OBJECTS</B> You can use the ``gsutil mv`` command to rename all objects with a given prefix to have a new prefix. For example, the following command renames all objects under gs://my_bucket/oldprefix to be under gs://my_bucket/newprefix, otherwise preserving the naming structure: gsutil mv gs://my_bucket/oldprefix gs://my_bucket/newprefix If you do a rename as specified above and you want to preserve ACLs, you should use the ``-p`` option (see OPTIONS). If you have a large number of files to move you might want to use the ``gsutil -m`` option, to perform a multi-threaded/multi-processing move: gsutil -m mv gs://my_bucket/oldprefix gs://my_bucket/newprefix <B>NON-ATOMIC OPERATION</B> Unlike the case with many file systems, the gsutil mv command does not perform a single atomic operation. Rather, it performs a copy from source to destination followed by removing the source for each object. A consequence of this is that, in addition to normal network and operation charges, if you move a Nearline Storage, Coldline Storage, or Archive Storage object, deletion and data retrieval charges apply. See the `documentation <https://cloud.google.com/storage/pricing>`_ for pricing details. <B>OPTIONS</B> All options that are available for the gsutil cp command are also available for the gsutil mv command (except for the -R flag, which is implied by the ``gsutil mv`` command). Please see the `Options section for cp <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for more information. """) class MvCommand(Command): """Implementation of gsutil mv command. Note that there is no atomic rename operation - this command is simply a shorthand for 'cp' followed by 'rm'. """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mv', command_name_aliases=['move', 'ren', 'rename'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # Flags for mv are passed through to cp. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mv', help_name_aliases=['move', 'rename'], help_type='command_help', help_one_line_summary='Move/rename objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'mv'], flag_map=CP_AND_MV_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the mv command.""" # Check each source arg up, refusing to delete a bucket src URL (force users # to explicitly do that as a separate operation). for arg_to_check in self.args[0:-1]: url = StorageUrlFromString(arg_to_check) if url.IsCloudUrl() and (url.IsBucket() or url.IsProvider()): raise CommandException('You cannot move a source bucket using the mv ' 'command. If you meant to move\nall objects in ' 'the bucket, you can use a command like:\n' '\tgsutil mv %s/* %s' % (arg_to_check, self.args[-1])) # Insert command-line opts in front of args so they'll be picked up by cp # and rm commands (e.g., for -p option). Use undocumented (internal # use-only) cp -M option, which causes each original object to be deleted # after successfully copying to its destination, and also causes naming # behavior consistent with Unix mv naming behavior (see comments in # ConstructDstUrl). unparsed_args = ['-M'] if self.recursion_requested: unparsed_args.append('-R') unparsed_args.extend(self.unparsed_args) self.command_runner.RunNamedCommand( 'cp', args=unparsed_args, headers=self.headers, debug=self.debug, trace_token=self.trace_token, user_project=self.user_project, parallel_operations=self.parallel_operations) return 0 defacl.py 0000644 00000035036 15025166704 0006351 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of default object acl command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib import gcs_json_api from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL _SET_SYNOPSIS = """ gsutil defacl set (<file-path>|<predefined-acl>) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defacl get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil defacl ch [-f] -u|-g|-d|-p <grant>... gs://<bucket_name>... """ _SET_DESCRIPTION = """ <B>SET</B> The ``defacl set`` command sets default object ACLs for the specified buckets. If you specify a default object ACL for a certain bucket, Cloud Storage applies the default object ACL to all new objects uploaded to that bucket, unless an ACL for that object is separately specified during upload. Similar to the ``acl set`` command, the ``defacl set`` command specifies either a predefined ACL or the path to a file that contains ACL text. See "gsutil help acl" for examples of editing and setting ACLs via the acl command. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. Setting a default object ACL on a bucket provides a convenient way to ensure newly uploaded objects have a specific ACL. If you don't set the bucket's default object ACL, it will default to project-private. If you then upload objects that need a different ACL, you will need to perform a separate ACL update operation for each object. Depending on how many objects require updates, this could be very time-consuming. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default ACL text for a bucket, which you can save and edit for use with the "defacl set" command. """ _CH_DESCRIPTION = """ <B>CH</B> The "defacl ch" (or "defacl change") command updates the default object access control list for a bucket. The syntax is shared with the "acl ch" command, so see the "CH" section of "gsutil help acl" for the full help description. <B>CH EXAMPLES</B> Grant anyone on the internet READ access by default to any object created in the bucket example-bucket: gsutil defacl ch -u AllUsers:R gs://example-bucket NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Add the user john.doe@example.com to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -u john.doe@example.com:READ gs://example-bucket Add the group admins@example.com to the default object ACL on bucket example-bucket with OWNER access: gsutil defacl ch -g admins@example.com:O gs://example-bucket Remove the group admins@example.com from the default object ACL on bucket example-bucket: gsutil defacl ch -d admins@example.com gs://example-bucket Add the owners of project example-project-123 to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -p owners-example-project-123:R gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ The defacl command has three sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION + _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class DefAclCommand(Command): """Implementation of gsutil defacl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defacl', command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='fg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defacl', help_name_aliases=['default acl', 'setdefacl', 'getdefacl', 'chdefacl'], help_type='command_help', help_one_line_summary='Get, set, or change default ACL on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=multi(defaultObjectAcl:format=json)', '--raw' ], flag_map={}, ) elif sub_command == 'set': acl_file_or_predefined_acl = self.args.pop(0) if (os.path.isfile(acl_file_or_predefined_acl)): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-object-acl-file=' + acl_file_or_predefined_acl ], flag_map={}, ) else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--predefined-default-object-acl=' + predefined_acl ], flag_map={}, ) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-g': GcloudStorageFlag('--add-default-object-acl-grant'), '-p': GcloudStorageFlag('--add-default-object-acl-grant'), '-u': GcloudStorageFlag('--add-default-object-acl-grant'), '-d': GcloudStorageFlag('--remove-default-object-acl-grant'), '-f': GcloudStorageFlag('--continue-on-error'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set' or self.command_alias_used == 'setdefacl'): return 1 else: return 0 def _SetDefAcl(self): if not StorageUrlFromString(self.args[-1]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException: self._WarnServiceAccounts() raise def _GetDefAcl(self): if not StorageUrlFromString(self.args[0]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) self.GetAndPrintAcl(self.args[0]) def _ChDefAcl(self): """Parses options and changes default object ACLs on specified buckets.""" self.parse_versions = True self.changes = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-g': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) if o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) if o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) if o == '-d': self.changes.append(acl_helper.AclDel(a)) if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) bucket_urls = set() for url_arg in self.args: for result in self.WildcardIterator(url_arg): if not result.storage_url.IsBucket(): raise CommandException( 'The defacl ch command can only be applied to buckets.') bucket_urls.add(result.storage_url) for storage_url in bucket_urls: self.ApplyAclChanges(storage_url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, url): """Applies the changes in self.changes to the provided URL.""" bucket = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['defaultObjectAcl', 'metageneration']) # Default object ACLs can be blank if the ACL was set to private, or # if the user doesn't have permission. We warn about this with defacl get, # so just try the modification here and if the user doesn't have # permission they'll get an AccessDeniedException. current_acl = bucket.defaultObjectAcl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return if not current_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. current_acl.append(PRIVATE_DEFAULT_OBJ_ACL) try: preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(defaultObjectAcl=current_acl) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) self.logger.info('Updated default ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, defacl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, defacl_message, 'defacl', self.logger) return modification_count def RunCommand(self): """Command entry point for the defacl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) self.def_acl = True self.continue_on_error = False if action_subcommand == 'get': func = self._GetDefAcl elif action_subcommand == 'set': func = self._SetDefAcl elif action_subcommand in ('ch', 'change'): func = self._ChDefAcl else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help defacl".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) func() return 0 rpo.py 0000644 00000017543 15025166704 0005736 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the rpo command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap VALID_RPO_VALUES = ('ASYNC_TURBO', 'DEFAULT') VALID_RPO_VALUES_STRING = '({})'.format('|'.join(VALID_RPO_VALUES)) _SET_SYNOPSIS = """ gsutil rpo set {} gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _GET_SYNOPSIS = """ gsutil rpo get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``rpo set`` command configures turbo replication for dual-region Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use turbo replication: gsutil rpo set ASYNC_TURBO gs://redbucket gs://bluebucket Configure your buckets to NOT use turbo replication: gsutil rpo set DEFAULT gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``rpo get`` command returns the replication setting for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if your buckets are using turbo replication: gsutil rpo get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``rpo`` command is used to retrieve or configure the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_ of dual-region Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) GET_COMMAND = GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'list', '--format=value[separator=": "](format("gs://{}", name),' 'rpo.yesno(no="None"))', '--raw' ], flag_map={}) SET_COMMAND = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--recovery-point-objective' ], flag_map={}, ) class RpoCommand(Command): """Implements the gsutil rpo command.""" command_spec = Command.CreateCommandSpec( 'rpo', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'set': [ CommandArgument('mode', choices=list(VALID_RPO_VALUES)), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rpo', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure replication', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GET_COMMAND, 'set': SET_COMMAND, }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetRpo(self, blr): """Gets the rpo setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['rpo'], provider=bucket_url.scheme) rpo = bucket_metadata.rpo bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, rpo)) def _SetRpo(self, blr, rpo_value): """Sets the rpo setting for a bucket.""" bucket_url = blr.storage_url formatted_rpo_value = rpo_value if formatted_rpo_value not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for rpo set.' ' Should be one of {}'.format(VALID_RPO_VALUES_STRING)) bucket_metadata = apitools_messages.Bucket(rpo=formatted_rpo_value) self.logger.info('Setting rpo %s for %s' % (formatted_rpo_value, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['rpo'], provider=bucket_url.scheme) return 0 def _Rpo(self): """Handles rpo command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('rpo only supports get|set') subcommand_func = None subcommand_args = [] if subcommand == 'get': subcommand_func = self._GetRpo elif subcommand == 'set': subcommand_func = self._SetRpo setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) # TODO: Remove this as rpo should work for XML as well. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: # TODO: Make this more general when adding RPO support for the XML API. if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the rpo command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._Rpo() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) version.py 0000644 00000016171 15025166704 0006617 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil version command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import platform import re import sys import six import boto import crcmod import gslib from gslib.command import Command from gslib.utils import system_util from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import UTF8 from gslib.utils.hashing_helper import GetMd5 from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit _SYNOPSIS = """ gsutil version """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Prints information about the version of gsutil. <B>OPTIONS</B> -l Prints additional information, such as the version of Python being used, the version of the Boto library, a checksum of the code, the path to gsutil, and the path to gsutil's configuration file. """) class VersionCommand(Command): """Implementation of gsutil version command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'version', command_name_aliases=['ver'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='l', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='version', help_name_aliases=['ver'], help_type='command_help', help_one_line_summary='Print version info about gsutil', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the version command.""" long_form = False if self.sub_opts: for o, _ in self.sub_opts: if o == '-l': long_form = True config_paths = ', '.join(GetFriendlyConfigFilePaths()) shipped_checksum = gslib.CHECKSUM try: cur_checksum = self._ComputeCodeChecksum() except IOError: cur_checksum = 'MISSING FILES' if shipped_checksum == cur_checksum: checksum_ok_str = 'OK' else: checksum_ok_str = '!= %s' % shipped_checksum sys.stdout.write('gsutil version: %s\n' % gslib.VERSION) if long_form: long_form_output = ( 'checksum: {checksum} ({checksum_ok})\n' 'boto version: {boto_version}\n' 'python version: {python_version}\n' 'OS: {os_version}\n' 'multiprocessing available: {multiprocessing_available}\n' 'using cloud sdk: {cloud_sdk}\n' 'pass cloud sdk credentials to gsutil: {cloud_sdk_credentials}\n' 'config path(s): {config_paths}\n' 'gsutil path: {gsutil_path}\n' 'compiled crcmod: {compiled_crcmod}\n' 'installed via package manager: {is_package_install}\n' 'editable install: {is_editable_install}\n' 'shim enabled: {is_shim_enabled}\n') sys.stdout.write( long_form_output.format( checksum=cur_checksum, checksum_ok=checksum_ok_str, boto_version=boto.__version__, python_version=sys.version.replace('\n', ''), os_version='%s %s' % (platform.system(), platform.release()), multiprocessing_available=( CheckMultiprocessingAvailableAndInit().is_available), cloud_sdk=system_util.InvokedViaCloudSdk(), cloud_sdk_credentials=system_util.CloudSdkCredPassingEnabled(), config_paths=config_paths, gsutil_path=(GetCloudSdkGsutilWrapperScriptPath() or gslib.GSUTIL_PATH), compiled_crcmod=UsingCrcmodExtension(), is_package_install=gslib.IS_PACKAGE_INSTALL, is_editable_install=gslib.IS_EDITABLE_INSTALL, is_shim_enabled=boto.config.getbool('GSUtil', 'use_gcloud_storage', False))) return 0 def _ComputeCodeChecksum(self): """Computes a checksum of gsutil code. This checksum can be used to determine if users locally modified gsutil when requesting support. (It's fine for users to make local mods, but when users ask for support we ask them to run a stock version of gsutil so we can reduce possible variables.) Returns: MD5 checksum of gsutil code. """ if gslib.IS_PACKAGE_INSTALL: return 'PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMS' m = GetMd5() # Checksum gsutil and all .py files under gslib directory. files_to_checksum = [gslib.GSUTIL_PATH] for root, _, files in os.walk(gslib.GSLIB_DIR): for filepath in files: if filepath.endswith('.py'): files_to_checksum.append(os.path.join(root, filepath)) # Sort to ensure consistent checksum build, no matter how os.walk # orders the list. for filepath in sorted(files_to_checksum): if six.PY2: f = open(filepath, 'rb') content = f.read() content = re.sub(r'(\r\n|\r|\n)', b'\n', content) m.update(content) f.close() else: f = open(filepath, 'r', encoding=UTF8) content = f.read() content = re.sub(r'(\r\n|\r|\n)', '\n', content) m.update(content.encode(UTF8)) f.close() return m.hexdigest() def GetCloudSdkGsutilWrapperScriptPath(): """If gsutil was invoked via the Cloud SDK, find its gsutil wrapper script. Returns: (str) The path to the Cloud SDK's gsutil wrapper script, or an empty string if gsutil was not invoked via the Cloud SDK or the wrapper script could not be found at its expected path. """ # If running via the Cloud SDK, the "real" gsutil script was probably invoked # indirectly through gcloud's gsutil wrapper script (the former is at # <sdk-root>/platform/gsutil/gsutil, while the latter is located at # <sdk-root>/bin/gsutil). We should print the wrapper script's path if it # exists and we were invoked via the Cloud SDK. gsutil_path = gslib.GSUTIL_PATH if system_util.InvokedViaCloudSdk(): platform_path_suffix = os.path.join('platform', 'gsutil', 'gsutil') if gsutil_path.endswith(platform_path_suffix): bin_path = os.path.join( gsutil_path[0:gsutil_path.rfind(platform_path_suffix)], 'bin', 'gsutil', ) if os.path.exists(bin_path): return bin_path return '' bucketpolicyonly.py 0000644 00000021310 15025166704 0010520 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands import ubla from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.text_util import InsistOnOrOff _SET_SYNOPSIS = """ gsutil bucketpolicyonly set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil bucketpolicyonly get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``bucketpolicyonly set`` command enables or disables the uniform bucket-level access feature on Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil bucketpolicyonly set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil bucketpolicyonly set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``bucketpolicyonly get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket. <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil bucketpolicyonly get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The Bucket Policy Only feature is now known as `uniform bucket-level access <https://cloud.google.com/storage/docs/uniform-bucket-level-access>`_. The ``bucketpolicyonly`` command is still supported, but we recommend using the equivalent ``ubla`` `command <https://cloud.google.com/storage/docs/gsutil/commands/ubla>`_. The ``bucketpolicyonly`` command is used to retrieve or configure the uniform bucket-level access setting of Cloud Storage buckets. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class BucketPolicyOnlyCommand(Command): """Implements the gsutil bucketpolicyonly command.""" command_spec = Command.CreateCommandSpec( 'bucketpolicyonly', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='bucketpolicyonly', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) # The get subcommand works exactly like ubla except we want to replace # "Uniform bucket-level access" with the bpo text. gcloud_storage_map = ubla.UblaCommand.gcloud_storage_map format_flag = gcloud_storage_map.gcloud_command['get'].gcloud_command[3] gcloud_storage_map.gcloud_command['get'].gcloud_command[ 3] = format_flag.replace('Uniform bucket-level access', 'Bucket Policy Only') def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetBucketPolicyOnly(self, blr): """Gets the Bucket Policy Only setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration bucket_policy_only = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': bucket_policy_only.enabled } locked_time_line = '' if bucket_policy_only.lockedTime: fields['locked_time'] = bucket_policy_only.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if bucket_policy_only: print(('Bucket Policy Only setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(**fields)) def _SetBucketPolicyOnly(self, blr, setting_arg): """Sets the Bucket Policy Only setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Bucket Policy Only for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _BucketPolicyOnly(self): """Handles bucketpolicyonly command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('bucketpolicyonly only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetBucketPolicyOnly elif subcommand == 'set': subcommand_func = self._SetBucketPolicyOnly setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the bucketpolicyonly command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._BucketPolicyOnly() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) pap.py 0000644 00000022403 15025166704 0005705 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the pap command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil pap set (enforced|inherited) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil pap get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``pap set`` command configures public access prevention for Cloud Storage buckets. If you set a bucket to be ``inherited``, it uses public access prevention only if the bucket is subject to the `public access prevention <https://cloud.google.com/storage/docs/org-policy-constraints#public-access-prevention>`_ organization policy constraint. <B>SET EXAMPLES</B> Configure ``redbucket`` and ``bluebucket`` to use public access prevention: gsutil pap set enforced gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``pap get`` command returns public access prevention values for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if ``redbucket`` and ``bluebucket`` are using public access prevention: gsutil pap get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``pap`` command is used to retrieve or configure the `public access prevention <https://cloud.google.com/storage/docs/public-access-prevention>`_ setting of Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit enforced one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue _GCLOUD_LIST_FORMAT = ('--format=value[separator=": "]' + '(name.sub("' + shim_util.get_format_flag_caret() + '","gs://"),' + 'iamConfiguration.publicAccessPrevention.yesno(' + 'no="inherited"))') class PapCommand(Command): """Implements the gsutil pap command.""" command_spec = Command.CreateCommandSpec( 'pap', command_name_aliases=['publicaccessprevention'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['enforced', 'inherited']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='pap', help_name_aliases=['publicaccessprevention'], help_type='command_help', help_one_line_summary='Configure public access prevention', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_LIST_FORMAT, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'enforced': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--public-access-prevention', ], flag_map={}, ), 'inherited': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-public-access-prevention', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetPublicAccessPrevention(self, blr): """Gets the public access prevention setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration public_access_prevention = iam_config.publicAccessPrevention or 'inherited' bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, public_access_prevention)) def _SetPublicAccessPrevention(self, blr, setting_arg): """Sets the Public Access Prevention setting for a bucket enforced or inherited.""" bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.publicAccessPrevention = setting_arg bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) print('Setting Public Access Prevention %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Pap(self): """Handles pap command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('pap only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetPublicAccessPrevention elif subcommand == 'set': subcommand_func = self._SetPublicAccessPrevention setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the pap command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Pap() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) hmac.py 0000644 00000041234 15025166704 0006040 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Implementation of HMAC key management command for GCS. NOTE: Any modification to this file or corresponding HMAC logic should be submitted in its own PR and release to avoid concurrency issues in testing. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.project_id import PopulateProjectId from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import shim_util _CREATE_SYNOPSIS = """ gsutil hmac create [-p <project>] <service_account_email> """ _DELETE_SYNOPSIS = """ gsutil hmac delete [-p <project>] <access_id> """ _GET_SYNOPSIS = """ gsutil hmac get [-p <project>] <access_id> """ _LIST_SYNOPSIS = """ gsutil hmac list [-a] [-l] [-p <project>] [-u <service_account_email>] """ _UPDATE_SYNOPSIS = """ gsutil hmac update -s (ACTIVE|INACTIVE) [-e <etag>] [-p <project>] <access_id> """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The ``hmac create`` command creates an HMAC key for the specified service account: gsutil hmac create test.service.account@test_project.iam.gserviceaccount.com The secret key material is only available upon creation, so be sure to store the returned secret along with the access_id. <B>CREATE OPTIONS</B> The ``create`` sub-command has the following option -p <project> Specify the ID or number of the project in which to create a key. """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The ``hmac delete`` command permanently deletes the specified HMAC key: gsutil hmac delete GOOG56JBMFZX6PMPTQ62VD2 Note that keys must be updated to be in the ``INACTIVE`` state before they can be deleted. <B>DELETE OPTIONS</B> The ``delete`` sub-command has the following option -p <project> Specify the ID or number of the project from which to delete a key. """ _GET_DESCRIPTION = """ <B>GET</B> The ``hmac get`` command retrieves the specified HMAC key's metadata: gsutil hmac get GOOG56JBMFZX6PMPTQ62VD2 Note that there is no option to retrieve a key's secret material after it has been created. <B>GET OPTIONS</B> The ``get`` sub-command has the following option -p <project> Specify the ID or number of the project from which to get a key. """ _LIST_DESCRIPTION = """ <B>LIST</B> The ``hmac list`` command lists the HMAC key metadata for keys in the specified project. If no project is specified in the command, the default project is used. <B>LIST OPTIONS</B> The ``list`` sub-command has the following options -a Show all keys, including recently deleted keys. -l Use long listing format. Shows each key's full metadata excluding the secret. -p <project> Specify the ID or number of the project from which to list keys. -u <service_account_email> Filter keys for a single service account. """ _UPDATE_DESCRIPTION = """ <B>UPDATE</B> The ``hmac update`` command sets the state of the specified key: gsutil hmac update -s INACTIVE -e M42da= GOOG56JBMFZX6PMPTQ62VD2 Valid state arguments are ``ACTIVE`` and ``INACTIVE``. To set a key to state ``DELETED``, use the ``hmac delete`` command on an ``INACTIVE`` key. If an etag is set in the command, it will only succeed if the provided etag matches the etag of the stored key. <B>UPDATE OPTIONS</B> The ``update`` sub-command has the following options -s <ACTIVE|INACTIVE> Sets the state of the specified key to either ``ACTIVE`` or ``INACTIVE``. -e <etag> If provided, the update will only be performed if the specified etag matches the etag of the stored key. -p <project> Specify the ID or number of the project in which to update a key. """ _SYNOPSIS = (_CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _GET_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _UPDATE_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ You can use the ``hmac`` command to interact with service account `HMAC keys <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. The ``hmac`` command has five sub-commands: """ + '\n'.join([ _CREATE_DESCRIPTION, _DELETE_DESCRIPTION, _GET_DESCRIPTION, _LIST_DESCRIPTION, _UPDATE_DESCRIPTION, ]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _VALID_UPDATE_STATES = ['INACTIVE', 'ACTIVE'] _TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' _create_help_text = CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION) _delete_help_text = CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _list_help_text = CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION) _update_help_text = CreateHelpText(_UPDATE_SYNOPSIS, _UPDATE_DESCRIPTION) def _AccessIdException(command_name, subcommand, synopsis): return CommandException( '%s %s requires an Access ID to be specified as the last argument.\n%s' % (command_name, subcommand, synopsis)) def _KeyMetadataOutput(metadata): """Format the key metadata for printing to the console.""" def FormatInfo(name, value, new_line=True): """Format the metadata name-value pair into two aligned columns.""" width = 22 info_str = '\t%-*s %s' % (width, name + ':', value) if new_line: info_str += '\n' return info_str message = 'Access ID %s:\n' % metadata.accessId message += FormatInfo('State', metadata.state) message += FormatInfo('Service Account', metadata.serviceAccountEmail) message += FormatInfo('Project', metadata.projectId) message += FormatInfo('Time Created', metadata.timeCreated.strftime(_TIME_FORMAT)) message += FormatInfo('Time Last Updated', metadata.updated.strftime(_TIME_FORMAT)) message += FormatInfo('Etag', metadata.etag, new_line=False) return message _CREATE_COMMAND_FORMAT = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("Access ID: {}", metadata.accessId),' + 'format("Secret: {}", secret))') _DESCRIBE_COMMAND_FORMAT = ( '--format=value[separator="' + shim_util.get_format_flag_newline() + '"](format("Access ID {}:", accessId),' + 'format("\tState: {}", state),' + 'format("\tService Account: {}", serviceAccountEmail),' + 'format("\tProject: {}", projectId),' + 'format("\tTime Created: {}",' + ' timeCreated.date(format="%a %d %b %Y %H:%M:%S GMT")),' + 'format("\tTime Last Updated: {}",' + ' updated.date(format="%a %d %b %Y %H:%M:%S GMT")),' + 'format("\tEtag: {}", etag))') _LIST_COMMAND_SHORT_FORMAT = ( '--format=table[no-heading](format("{} ", accessId),' 'state:width=11, serviceAccountEmail)') _PROJECT_FLAG = GcloudStorageFlag('--project') CREATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'create', _CREATE_COMMAND_FORMAT], flag_map={ '-p': _PROJECT_FLAG, }) DELETE_COMMAND = GcloudStorageMap(gcloud_command=['storage', 'hmac', 'delete'], flag_map={ '-p': _PROJECT_FLAG, }) GET_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'describe', _DESCRIBE_COMMAND_FORMAT], flag_map={'-p': _PROJECT_FLAG}) LIST_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _LIST_COMMAND_SHORT_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) LIST_COMMAND_LONG_FORMAT = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-l': GcloudStorageFlag('--long'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) UPDATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'update', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-s': GcloudStorageFlag({ 'ACTIVE': '--activate', 'INACTIVE': '--deactivate', }), '-e': GcloudStorageFlag('--etag'), '-p': _PROJECT_FLAG }) class HmacCommand(Command): """Implementation of gsutil hmac command.""" command_spec = Command.CreateCommandSpec( 'hmac', min_args=1, max_args=8, supported_sub_args='ae:lp:s:u:', file_url_ok=True, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, usage_synopsis=_SYNOPSIS, argparse_arguments={ 'create': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'delete': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'get': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'list': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'update': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], }, ) help_spec = Command.HelpSpec( help_name='hmac', help_name_aliases=[], help_type='command_help', help_one_line_summary=('CRUD operations on service account HMAC keys.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'delete': _delete_help_text, 'get': _get_help_text, 'list': _list_help_text, 'update': _update_help_text, }) def get_gcloud_storage_args(self): if self.args[0] == 'list' and '-l' in self.args: gcloud_storage_map = GcloudStorageMap( gcloud_command={'list': LIST_COMMAND_LONG_FORMAT}, flag_map={}, ) else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': CREATE_COMMAND, 'delete': DELETE_COMMAND, 'update': UPDATE_COMMAND, 'get': GET_COMMAND, 'list': LIST_COMMAND }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _CreateHmacKey(self, thread_state=None): """Creates HMAC key for a service account.""" if self.args: self.service_account_email = self.args[0] else: err_msg = ('%s %s requires a service account to be specified as the ' 'last argument.\n%s') raise CommandException( err_msg % (self.command_name, self.action_subcommand, _CREATE_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.CreateHmacKey(self.project_id, self.service_account_email, provider='gs') print('%-12s %s' % ('Access ID:', response.metadata.accessId)) print('%-12s %s' % ('Secret:', response.secret)) def _DeleteHmacKey(self, thread_state=None): """Deletes an HMAC key.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _DELETE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) gsutil_api.DeleteHmacKey(self.project_id, access_id, provider='gs') def _GetHmacKey(self, thread_state=None): """Gets HMAC key from its Access Id.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _GET_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.GetHmacKey(self.project_id, access_id, provider='gs') print(_KeyMetadataOutput(response)) def _ListHmacKeys(self, thread_state=None): """Lists HMAC keys for a project or service account.""" if self.args: raise CommandException( '%s %s received unexpected arguments.\n%s' % (self.command_name, self.action_subcommand, _LIST_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.ListHmacKeys(self.project_id, self.service_account_email, self.show_all, provider='gs') short_list_format = '%s\t%-12s %s' if self.long_list: for item in response: print(_KeyMetadataOutput(item)) print() else: for item in response: print(short_list_format % (item.accessId, item.state, item.serviceAccountEmail)) def _UpdateHmacKey(self, thread_state=None): """Update an HMAC key's state.""" if not self.state: raise CommandException( 'A state flag must be supplied for %s %s\n%s' % (self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS)) elif self.state not in _VALID_UPDATE_STATES: raise CommandException('The state flag value must be one of %s' % ', '.join(_VALID_UPDATE_STATES)) if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.UpdateHmacKey(self.project_id, access_id, self.state, self.etag, provider='gs') print(_KeyMetadataOutput(response)) def RunCommand(self): """Command entry point for the hmac command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException( 'The "hmac" command can only be used with the GCS JSON API') self.action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.service_account_email = None self.state = None self.show_all = False self.long_list = False self.etag = None if self.sub_opts: for o, a in self.sub_opts: if o == '-u': self.service_account_email = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-s': self.state = a elif o == '-a': self.show_all = True elif o == '-l': self.long_list = True elif o == '-e': self.etag = a if not self.project_id: self.project_id = PopulateProjectId(None) method_for_arg = { 'create': self._CreateHmacKey, 'delete': self._DeleteHmacKey, 'get': self._GetHmacKey, 'list': self._ListHmacKeys, 'update': self._UpdateHmacKey, } if self.action_subcommand not in method_for_arg: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help hmac".' % (self.action_subcommand, self.command_name)) LogCommandParams(subcommands=[self.action_subcommand]) method_for_arg[self.action_subcommand]() return 0 rb.py 0000644 00000013326 15025166704 0005534 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rb command for deleting cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.cloud_api import NotEmptyException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil rb [-f] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Delete one or more buckets. Buckets must be empty before you can delete them. Be certain you want to delete a bucket before you do so, as once it is deleted the name becomes available and another user may create a bucket with that name. (But see also "DOMAIN NAMED BUCKETS" under "gsutil help naming" for help carving out parts of the bucket name space.) <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing buckets. If some buckets couldn't be removed, gsutil's exit status will be non-zero even if this flag is set. If no buckets could be removed, the command raises a "no matches" error. """) class RbCommand(Command): """Implementation of gsutil rb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rb', command_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='f', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rb', help_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], help_type='command_help', help_one_line_summary='Remove buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'delete'], flag_map={ '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rb command.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': self.continue_on_error = True did_some_work = False some_failed = False for url_str in self.args: wildcard_url = StorageUrlFromString(url_str) if wildcard_url.IsObject(): raise CommandException('"rb" command requires a provider or ' 'bucket URL') # Wrap WildcardIterator call in try/except so we can avoid printing errors # with -f option if a non-existent URL listed, permission denial happens # while listing, etc. try: # Materialize iterator results into a list to catch exceptions. # Since this is listing buckets this list shouldn't be too large to fit # in memory at once. # Also, avoid listing all fields to avoid performing unnecessary bucket # metadata GETs. These would also be problematic when billing is # disabled, as deletes are allowed but GetBucket is not. blrs = list( self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id'])) except: # pylint: disable=bare-except some_failed = True if self.continue_on_error: continue else: raise for blr in blrs: url = blr.storage_url self.logger.info('Removing %s...', url) try: self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) except NotEmptyException as e: some_failed = True if self.continue_on_error: continue elif 'VersionedBucketNotEmpty' in e.reason: raise CommandException('Bucket is not empty. Note: this is a ' 'versioned bucket, so to delete all ' 'objects\nyou need to use:' '\n\tgsutil rm -r %s' % url) else: raise except: # pylint: disable=bare-except some_failed = True if not self.continue_on_error: raise did_some_work = True if not did_some_work: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 1 if some_failed else 0 stat.py 0000644 00000015203 15025166704 0006100 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like stat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import sys from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil stat url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The stat command outputs `object metadata <https://cloud.google.com/storage/docs/metadata>`_ for the specified object URLs. It is similar to running: gsutil ls -L gs://some-bucket/some-object but is more efficient because it avoids listing objects and gets the minimum necessary amount of object metadata. Moreover, because it avoids listing objects (which for some storage providers are eventually consistent) the ``gsutil stat`` command provides a strongly consistent way to check for the existence (and read the metadata) of an object. The ``gsutil stat`` command does, however, make listing object requests if you specify URLs using wildcards. If run with the ``gsutil -q`` option nothing is printed, e.g.: gsutil -q stat gs://some-bucket/some-object This behavior can be useful when writing scripts: even though nothing is printed from the command, it still has an exit status of 0 for an existing object and 1 for a non-existent object. NOTE: Unlike the gsutil ls command, the stat command does not support operations on sub-directories. For example, if you run the command: gsutil -q stat gs://some-bucket/some-subdir/ gsutil looks for information about an object called "some-subdir/" (with a trailing slash) inside the bucket "some-bucket", as opposed to operating on objects nested under gs://some-bucket/some-subdir/. Unless you actually have an object with that name, the operation fails. However, you can use the stat command on objects within subdirectories. For example, this command works as expected: gsutil -q stat gs://some-bucket/some-subdir/file.txt """) # TODO: Add ability to stat buckets. class StatCommand(Command): """Implementation of gsutil stat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'stat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='stat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object status', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=[ # Formatting done in gcloud. 'storage', 'objects', 'list', '--fetch-encrypted-object-hashes', '--stat', ], flag_map={}, ) def RunCommand(self): """Command entry point for stat command.""" stat_fields = ENCRYPTED_FIELDS + UNENCRYPTED_FULL_LISTING_FIELDS found_nonmatching_arg = False for url_str in self.args: arg_matches = 0 url = StorageUrlFromString(url_str) if not url.IsObject(): raise CommandException('The stat command only works with object URLs') try: if ContainsWildcard(url_str): blr_iter = self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=stat_fields) else: try: single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=stat_fields) except EncryptionException: # Retry without requesting hashes. single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=UNENCRYPTED_FULL_LISTING_FIELDS) blr_iter = [BucketListingObject(url, root_object=single_obj)] for blr in blr_iter: if blr.IsObject(): arg_matches += 1 # TODO: Request fewer fields if we're not printing the object. if logging.getLogger().isEnabledFor(logging.INFO): PrintFullInfoAboutObject(blr, incl_acl=False) except AccessDeniedException: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write('You aren\'t authorized to read %s - skipping' % url_str + '\n') except InvalidUrlError: raise except NotFoundException: pass if not arg_matches: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write(NO_URLS_MATCHED_TARGET % url_str + '\n') found_nonmatching_arg = True if found_nonmatching_arg: return 1 return 0 cors.py 0000644 00000021054 15025166704 0006074 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of cors configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import CorsTranslation from gslib.utils.translation_helper import REMOVE_CORS_CONFIG _GET_SYNOPSIS = """ gsutil cors get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil cors set <cors-json-file> gs://<bucket_name>... """ _GET_DESCRIPTION = """ <B>GET</B> Gets the CORS configuration for a single bucket. The output from ``cors get`` can be redirected into a file, edited and then updated using ``cors set``. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the CORS configuration for one or more buckets. The ``cors-json-file`` specified on the command line should be a path to a local file containing a JSON-formatted CORS configuration, such as the example described above. """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n\n' _DESCRIPTION = (""" Gets or sets the Cross-Origin Resource Sharing (CORS) configuration on one or more buckets. This command is supported for buckets only, not objects. An example CORS JSON file looks like the following: [ { "origin": ["http://origin1.example.com"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 3600 } ] The above CORS configuration explicitly allows cross-origin GET requests from http://origin1.example.com and may include the Content-Type response header. The preflight request may be cached for 1 hour. Note that requests to the authenticated browser download endpoint ``storage.cloud.google.com`` do not allow CORS requests. For more information about supported endpoints for CORS, see `Cloud Storage CORS support <https://cloud.google.com/storage/docs/cross-origin#server-side-support>`_. The following (empty) CORS JSON file removes any CORS configuration for a bucket: [] The cors command has two sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION]) + """ For more info about CORS generally, see https://www.w3.org/TR/cors/. For more info about CORS in Cloud Storage, see the `CORS concept page <https://cloud.google.com/storage/docs/cross-origin>`_. """) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class CorsCommand(Command): """Implementation of gsutil cors command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cors', command_name_aliases=['getcors', 'setcors'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cors', help_name_aliases=[ 'getcors', 'setcors', 'cross-origin', ], help_type='command_help', help_one_line_summary=( 'Get or set a CORS configuration for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=cors_config,empty=\' has no ' 'CORS configuration.\',empty_prefix_key=storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--cors-file' ], flag_map={}, ), }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetCors(self): """Sets CORS configuration on a Google Cloud Storage bucket.""" cors_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'cors set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. cors_file = open(cors_arg, 'r') cors_txt = cors_file.read() cors_file.close() self.api = self.gsutil_api.GetApiSelector( StorageUrlFromString(url_args[0]).scheme) # Iterate over URLs, expanding wildcards and setting the CORS on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting CORS on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetCors(cors_txt, url, provider=url.scheme) else: cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt) if not cors: cors = REMOVE_CORS_CONFIG bucket_metadata = apitools_messages.Bucket(cors=cors) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetCors(self): """Gets CORS configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['cors']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetCors(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.cors: sys.stdout.write( CorsTranslation.MessageEntriesToJson(bucket_metadata.cors)) else: sys.stdout.write('%s has no CORS configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the cors command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetCors elif action_subcommand == 'set': func = self._SetCors else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help cors".') % (action_subcommand, self.command_name)) metrics.LogCommandParams(subcommands=[action_subcommand]) return func() config.py 0000644 00000146223 15025166704 0006401 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of config command for creating a gsutil configuration file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import json import multiprocessing import os import signal import socket import stat import sys import textwrap import time import webbrowser from six.moves import input from six.moves.http_client import ResponseNotReady import boto from boto.provider import Provider import gslib from gslib.command import Command from gslib.command import DEFAULT_TASK_ESTIMATION_THRESHOLD from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.cred_types import CredTypes from gslib.exception import AbortException from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import constants from gslib.utils import system_util from gslib.utils.hashing_helper import CHECK_HASH_ALWAYS from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_SKIP from gslib.utils.hashing_helper import CHECK_HASH_NEVER from gslib.utils.parallelism_framework_util import ShouldProhibitMultiprocessing from httplib2 import ServerNotFoundError from oauth2client.client import HAS_CRYPTO _SYNOPSIS = """ gsutil [-D] config (-a|-e) [-n] [-o <file>] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil config`` command generally applies to users who have legacy standalone installations of gsutil. If you installed gsutil via the Cloud SDK, ``gsutil config`` fails unless you are specifically using the ``-a`` flag or have configured gcloud to not pass its managed credentials to gsutil (via the command ``gcloud config set pass_credentials_to_gsutil false``). For all other use cases, Cloud SDK users should use the ``gcloud auth`` group of commands instead, which configures OAuth2 credentials that gcloud implicitly passes to gsutil at runtime. To check if you are using gsutil from the Cloud SDK or as a legacy standalone, use ``gsutil version -l`` and in the output look for "using cloud sdk". Important: The default behavior for the ``gsutil config`` command is to obtain user account credentials for authentication. However, user account credentials are no longer supported for standalone gsutil. For this reason, running the default ``gsutil config`` command fails, and using any of the following flags causes the command to fail: ``-b``, ``-f``, ``-r``, ``--reauth``, ``-s``, ``-w``. When using standalone gsutil, it's recommended that you use service account credentials via the ``-e`` flag. The ``gsutil config`` command obtains access credentials for Cloud Storage and writes a `boto/gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ containing the obtained credentials along with a number of other configuration- controllable values. Unless specified otherwise (see OPTIONS), the configuration file is written to ~/.boto (i.e., the file .boto under the user's home directory). If the default file already exists, an attempt is made to rename the existing file to ~/.boto.bak; if that attempt fails the command exits. A different destination file can be specified with the ``-o`` option (see OPTIONS). Because the boto configuration file contains your credentials you should keep its file permissions set so no one but you has read access. (The file is created read-only when you run ``gsutil config``.) <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> Service accounts are useful for authenticating on behalf of a service or application (as opposed to a user). If you use gsutil as a legacy stand-alone tool, you configure credentials for service accounts using the ``-e`` option: gsutil config -e Note that if you use gsutil through the Cloud SDK, you instead activate your service account via the `gcloud auth activate-service-account <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ command. When you run ``gsutil config -e``, you are prompted for the path to your private key file and, if not using a JSON key file, your service account email address and key file password. To get this data, follow the instructions on `Service Accounts <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_. Using this information, gsutil populates the "gs_service_key_file" attribute in the boto configuration file. If not using a JSON key file, gsutil also populates the "gs_service_client_id" and "gs_service_key_file_password" attributes. Note that your service account is NOT considered an Owner for the purposes of API access (see "gsutil help creds" for more information about this). See https://developers.google.com/identity/protocols/OAuth2ServiceAccount for further information on service account authentication. If you want to use credentials based on access key and secret (the older authentication method before OAuth2 was supported), see the ``-a`` option in the OPTIONS section. If you wish to use gsutil with other providers (or to copy data back and forth between multiple providers) you can edit their credentials into the [Credentials] section after creating the initial boto configuration file. <B>OPTIONS</B> -a Prompt for Google Cloud Storage access key and secret (the older authentication method before OAuth2 was supported) instead of obtaining an OAuth2 token. -e Prompt for service account credentials. This option requires that ``-a`` is not set. -n Write the configuration file without authentication configured. This flag is mutually exclusive with all flags other than ``-o``. -o <file> Write the configuration to <file> instead of ~/.boto. Use ``-`` for stdout. """) GOOG_CLOUD_CONSOLE_URI = 'https://cloud.google.com/console#/project' CONFIG_PRELUDE_CONTENT = """ # This file contains credentials and other configuration information needed # by the boto library, used by gsutil. You can edit this file (e.g., to add # credentials) but be careful not to mis-edit any of the variable names (like # "gs_access_key_id") or remove important markers (like the "[Credentials]" and # "[Boto]" section delimiters). # """ # Default number of OS processes and Python threads for parallel operations. # On Linux systems we automatically scale the number of processes to match # the underlying CPU/core count. Given we'll be running multiple concurrent # processes on a typical multi-core Linux computer, to avoid being too # aggressive with resources, the default number of threads is reduced from # the previous value of 24 to 5. # # We also cap the maximum number of default processes at 32. Since each level # of recursion depth gets its own process pool, this means a maximum of # 64 processes with the current maximum recursion depth of 2. We limit this # number because testing with more than 200 processes showed fatal locking # exceptions in Python's multiprocessing.Manager. More processes are # probably not needed to saturate most networks. # # On Windows and Alpine Linux, Python multi-processing presents various # challenges so we retain compatibility with the established parallel mode # operation, i.e. one process and 24 threads. should_prohibit_multiprocessing, unused_os = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing: DEFAULT_PARALLEL_PROCESS_COUNT = 1 DEFAULT_PARALLEL_THREAD_COUNT = 24 else: DEFAULT_PARALLEL_PROCESS_COUNT = min(multiprocessing.cpu_count(), 32) DEFAULT_PARALLEL_THREAD_COUNT = 5 # TODO: Once compiled crcmod is being distributed by major Linux distributions # revert DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD value to '150M'. DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD = '0' DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE = '50M' DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD = '150M' DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE = '200M' DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS = 4 # Compressed transport encoded uploads buffer chunks of compressed data. When # running many uploads in parallel, compression may consume more memory than # available. This restricts the number of compressed transport encoded uploads # running in parallel such that they don't consume more memory than set here. DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = '2G' # gzip compression level. This is simply making the python stdlib default explicit. DEFAULT_GZIP_COMPRESSION_LEVEL = 9 CONFIG_BOTO_SECTION_CONTENT = """ [Boto] # http_socket_timeout specifies the timeout (in seconds) used to tell httplib # how long to wait for socket timeouts. The default is 70 seconds. Note that # this timeout only applies to httplib, not to httplib2 (which is used for # OAuth2 refresh/access token exchanges). #http_socket_timeout = 70 # The following two options control the use of a secure transport for requests # to S3 and Google Cloud Storage. It is highly recommended to set both options # to True in production environments, especially when using OAuth2 bearer token # authentication with Google Cloud Storage. # Set 'https_validate_certificates' to False to disable server certificate # checking. The default for this option in the boto library is currently # 'False' (to avoid breaking apps that depend on invalid certificates); it is # therefore strongly recommended to always set this option explicitly to True # in configuration files, to protect against "man-in-the-middle" attacks. https_validate_certificates = True # 'debug' controls the level of debug messages printed for the XML API only: # 0 for none, 1 for basic boto debug, 2 for all boto debug plus HTTP # requests/responses. #debug = <0, 1, or 2> # 'num_retries' controls the number of retry attempts made when errors occur # during data transfers. The default is 6. # Note 1: You can cause gsutil to retry failures effectively infinitely by # setting this value to a large number (like 10000). Doing that could be useful # in cases where your network connection occasionally fails and is down for an # extended period of time, because when it comes back up gsutil will continue # retrying. However, in general we recommend not setting the value above 10, # because otherwise gsutil could appear to "hang" due to excessive retries # (since unless you run gsutil -D you won't see any logged evidence that gsutil # is retrying). # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing # HTTP connections. #num_retries = <integer value> # 'max_retry_delay' controls the max delay (in seconds) between retries. The # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. # Note: At present this value only impacts the XML API and the JSON API uses a # fixed value of 60. #max_retry_delay = <integer value> """ CONFIG_GOOGLECOMPUTE_SECTION_CONTENT = """ [GoogleCompute] # 'service_account' specifies the a Google Compute Engine service account to # use for credentials. This value is intended for use only on Google Compute # Engine virtual machines and usually lives in /etc/boto.cfg. Most users # shouldn't need to edit this part of the config. #service_account = default """ CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT = """ [GSUtil] # 'resumable_threshold' specifies the smallest file size [bytes] for which # resumable Google Cloud Storage uploads are attempted. The default is 8388608 # (8 MiB). #resumable_threshold = %(resumable_threshold)d # 'rsync_buffer_lines' specifies the number of lines of bucket or directory # listings saved in each temp file during sorting. (The complete set is # split across temp files and separately sorted/merged, to avoid needing to # fit everything in memory at once.) If you are trying to synchronize very # large directories/buckets (e.g., containing millions or more objects), # having too small a value here can cause gsutil to run out of open file # handles. If that happens, you can try to increase the number of open file # handles your system allows (e.g., see 'man ulimit' on Linux; see also # http://docs.python.org/2/library/resource.html). If you can't do that (or # if you're already at the upper limit), increasing rsync_buffer_lines will # cause gsutil to use fewer file handles, but at the cost of more memory. With # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes # long, gsutil will require approximately 10 MiB of memory while building # the synchronization state, and will require approximately 60 open file # descriptors to build the synchronization state over all 1M source and 1M # destination URLs. Memory and file descriptors are only consumed while # building the state; once the state is built, it resides in two temp files that # are read and processed incrementally during the actual copy/delete # operations. #rsync_buffer_lines = 32000 # 'state_dir' specifies the base location where files that # need a static location are stored, such as pointers to credentials, # resumable transfer tracker files, and the last software update check. # By default these files are stored in ~/.gsutil #state_dir = <file_path> # gsutil periodically checks whether a new version of the gsutil software is # available. 'software_update_check_period' specifies the number of days # between such checks. The default is 30. Setting the value to 0 disables # periodic software update checks. #software_update_check_period = 30 # 'tab_completion_timeout' controls the timeout (in seconds) for tab # completions that involve remote requests (such as bucket or object names). # If tab completion does not succeed within this timeout, no tab completion # suggestions will be returned. # A value of 0 will disable completions that involve remote requests. #tab_completion_timeout = 5 # 'parallel_process_count' and 'parallel_thread_count' specify the number # of OS processes and Python threads, respectively, to use when executing # operations in parallel. The default settings should work well as configured, # however, to enhance performance for transfers involving large numbers of # files, you may experiment with hand tuning these values to optimize # performance for your particular system configuration. #parallel_process_count = %(parallel_process_count)d #parallel_thread_count = %(parallel_thread_count)d # 'parallel_composite_upload_threshold' specifies the maximum size of a file to # upload in a single stream. Files larger than this threshold will be # partitioned into component parts and uploaded in parallel and then composed # into a single object. # The number of components will be the smaller of # ceil(file_size / parallel_composite_upload_component_size) and # MAX_COMPOSE_ARITY. The current value of MAX_COMPOSE_ARITY is # %(max_compose_arity)d. # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel # uploads will never occur. # Setting an extremely low threshold is unadvisable. The vast majority of # environments will see degraded performance for thresholds below 80M, and it # is almost never advantageous to have a threshold below 20M. # 'parallel_composite_upload_component_size' specifies the ideal size of a # component in bytes, which will act as an upper bound to the size of the # components if ceil(file_size / parallel_composite_upload_component_size) is # less than MAX_COMPOSE_ARITY. # Values can be provided either in bytes or as human-readable values # (e.g., "150M" to represent 150 mebibytes) # # Note: At present parallel composite uploads are disabled by default, because # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), # and for operating systems that don't already have this package installed this # makes gsutil harder to use. Google is actively working with a number of the # Linux distributions to get crcmod included with the stock distribution. Once # that is done we will re-enable parallel composite uploads by default in # gsutil. # # Note: Parallel composite uploads should not be used with NEARLINE, COLDLINE, # or ARCHIVE storage class buckets, as doing this incurs an early deletion # charge for each component object. # # Note: Parallel composite uploads are not enabled with Cloud KMS encrypted # objects as a source or destination, as composition with KMS objects is not yet # supported. #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s #parallel_composite_upload_component_size = %(parallel_composite_upload_component_size)s # # 'parallel_composite_upload_bypass_kms_check' removes the object/bucket KMS checks # used to guard composition of KMS objects. #disable_parallel_composite_upload_kms_check = False # 'sliced_object_download_threshold' and # 'sliced_object_download_component_size' have analogous functionality to # their respective parallel_composite_upload config values. # 'sliced_object_download_max_components' specifies the maximum number of # slices to be used when performing a sliced object download. #sliced_object_download_threshold = %(sliced_object_download_threshold)s #sliced_object_download_component_size = %(sliced_object_download_component_size)s #sliced_object_download_max_components = %(sliced_object_download_max_components)s # Compressed transport encoded uploads buffer chunks of compressed data. When # running a composite upload and/or many uploads in parallel, compression may # consume more memory than available. This setting restricts the number of # compressed transport encoded uploads running in parallel such that they # don't consume more memory than set here. This is 2GiB by default. # Values can be provided either in bytes or as human-readable values # (e.g., "2G" to represent 2 gibibytes) #max_upload_compression_buffer_size = %(max_upload_compression_buffer_size)s # GZIP compression level, if using compression. Reducing this can have # a dramatic impact on compression speed with minor size increases. # This is a value from 0-9, with 9 being max compression. # A good level to try is 6, which is the default used by the gzip tool. #gzip_compression_level = %(gzip_compression_level)s # 'task_estimation_threshold' controls how many files or objects gsutil # processes before it attempts to estimate the total work that will be # performed by the command. Estimation makes extra directory listing or API # list calls and is performed only if multiple processes and/or threads are # used. Estimation can slightly increase cost due to extra # listing calls; to disable it entirely, set this value to 0. #task_estimation_threshold=%(task_estimation_threshold)s # 'use_magicfile' specifies if the 'file --mime <filename>' command should be # used to guess content types instead of the default filename extension-based # mechanism. Available on UNIX and macOS (and possibly on Windows, if you're # running Cygwin or some other package that provides implementations of # UNIX-like commands). When available and enabled use_magicfile should be more # robust because it analyzes file contents in addition to extensions. #use_magicfile = False # Service account emails for testing the hmac command. If these fields are not # populated with distinct service accounts the tests for the hmac command will # not be run. Primarily useful for tool developers. #test_hmac_service_account = #test_hmac_alt_service_account = #test_hmac_list_service_account = # Service account emails for testing impersonation credentials. If this field is # not populated with a service account the tests for service account # impersonation will not run. Primarily useful for tool developers. #test_impersonate_service_account = # 'content_language' specifies the ISO 639-1 language code of the content, to be # passed in the Content-Language header. By default no Content-Language is sent. # See the ISO 639-1 column of # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of # language codes. content_language = en # 'check_hashes' specifies how strictly to require integrity checking for # downloaded data. Legal values are: # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest # will run efficiently (using compiled code), else fail the download. # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a # hash and the local digest computation will run quickly, else skip the # check. # '%(hash_always)s' - Always check download integrity regardless of possible # performance costs. # '%(hash_never)s' - Don't perform download integrity checks. This setting is # not recommended except for special cases such as measuring download # performance excluding time for integrity checking. # This option exists to assist users who wish to download a GCS composite object # and are unable to install crcmod with the C-extension. CRC32c is the only # available integrity check for composite objects, and without the C-extension, # download performance can be significantly degraded by the digest computation. # This option is ignored for daisy-chain copies, which don't compute hashes but # instead (inexpensively) compare the cloud source and destination hashes. #check_hashes = if_fast_else_fail # 'encryption_key' specifies a single customer-supplied encryption key that # will be used for all data written to Google Cloud Storage. See # "gsutil help encryption" for more information # Encryption key: RFC 4648 section 4 base64-encoded AES256 string # Warning: If decrypt_key is specified without an encrypt_key, objects will be # decrypted when copied in the cloud. #encryption_key= # Each 'decryption_key' entry specifies a customer-supplied decryption key that # will be used to access and Google Cloud Storage objects encrypted with # the corresponding key. # Decryption keys: Up to 100 RFC 4648 section 4 base64-encoded AES256 strings # in ascending numerical order, starting with 1. #decryption_key1= #decryption_key2= #decryption_key3= # The ability to specify an alternative JSON API version is primarily for cloud # storage service developers. #json_api_version = v1 # Specifies the API to use when interacting with cloud storage providers. If the # gsutil command supports this API for the provider, it will be used instead of # the default API. Commands typically default to XML for S3 and JSON for GCS. # Note that if any encryption configuration options are set (see above), the # JSON API will be used for interacting with Google Cloud Storage buckets even # if XML is preferred, as gsutil does not currently support this functionality # when using the XML API. #prefer_api = json #prefer_api = xml # Disables the prompt asking for opt-in to data collection for analytics. #disable_analytics_prompt = True # The "test" command runs tests against regional buckets (unless you supply the # `-b` option). By default, the region used is us-central1, but you can change # the default region using this option. #test_cmd_regional_bucket_location = us-central1 # Tests for the "notification watchbucket" command require a notification URL. # If this option is not supplied, those tests will be skipped. #test_notification_url = https://yourdomain.url/notification-endpoint # Used in conjunction with --stet flag on cp command for end-to-end encryption. # STET binary path. If not specified, gsutil checks PATH for "stet". #stet_binary_path = <Path to binary "/usr/local/bin/stet"> # STET config path. If not specified, the STET binary will run with its default # settings. #stet_config_path = ~/.config/my_config.yaml # Adds an API call before parallel operations that triggers a reauth challenge. #trigger_reauth_challenge_for_parallel_operations = False """ % { 'hash_fast_else_fail': CHECK_HASH_IF_FAST_ELSE_FAIL, 'hash_fast_else_skip': CHECK_HASH_IF_FAST_ELSE_SKIP, 'hash_always': CHECK_HASH_ALWAYS, 'hash_never': CHECK_HASH_NEVER, 'resumable_threshold': constants.RESUMABLE_THRESHOLD_B, 'parallel_process_count': DEFAULT_PARALLEL_PROCESS_COUNT, 'parallel_thread_count': DEFAULT_PARALLEL_THREAD_COUNT, 'parallel_composite_upload_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'parallel_composite_upload_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'sliced_object_download_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_max_components': (DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS), 'max_compose_arity': MAX_COMPOSE_ARITY, 'task_estimation_threshold': DEFAULT_TASK_ESTIMATION_THRESHOLD, 'max_upload_compression_buffer_size': (DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE), 'gzip_compression_level': DEFAULT_GZIP_COMPRESSION_LEVEL, } CONFIG_OAUTH2_CONFIG_CONTENT = """ [OAuth2] # This section specifies options used with OAuth2 authentication. # 'token_cache' specifies how the OAuth2 client should cache access tokens. # Valid values are: # 'in_memory': an in-memory cache is used. This is only useful if the boto # client instance (and with it the OAuth2 plugin instance) persists # across multiple requests. # 'file_system' : access tokens will be cached in the file system, in files # whose names include a key derived from the refresh token the access token # based on. # The default is 'file_system'. #token_cache = file_system #token_cache = in_memory # 'token_cache_path_pattern' specifies a path pattern for token cache files. # This option is only relevant if token_cache = file_system. # The value of this option should be a path, with place-holders '%(key)s' (which # will be replaced with a key derived from the refresh token the cached access # token was based on), and (optionally), %(uid)s (which will be replaced with # the UID of the current user, if available via os.getuid()). # Note that the config parser itself interpolates '%' placeholders, and hence # the above placeholders need to be escaped as '%%(key)s'. # The default value of this option is # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s # where <tmpdir> is the system-dependent default temp directory. # The following options specify the label and endpoint URIs for the OAUth2 # authorization provider being used. Primarily useful for tool developers. #provider_label = Google #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth #provider_token_uri = https://oauth2.googleapis.com/token # 'oauth2_refresh_retries' controls the number of retry attempts made when # rate limiting errors occur for OAuth2 requests to retrieve an access token. # The default value is 6. #oauth2_refresh_retries = <integer value> # The following options specify the OAuth2 client identity and secret that is # used when requesting and using OAuth2 tokens. If not specified, a default # OAuth2 client for the gsutil tool is used; for uses of the boto library (with # OAuth2 authentication plugin) in other client software, it is recommended to # use a tool/client-specific OAuth2 client. For more information on OAuth2, see # http://code.google.com/apis/accounts/docs/OAuth2.html """ class ConfigCommand(Command): """Implementation of gsutil config command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'config', command_name_aliases=['cfg', 'conf', 'configure'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='abefhno:rs:w', supported_private_args=['reauth'], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='config', help_name_aliases=['cfg', 'conf', 'configure', 'aws', 's3'], help_type='command_help', help_one_line_summary=( 'Obtain credentials and create configuration file'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _OpenConfigFile(self, file_path): """Creates and opens a configuration file for writing. The file is created with mode 0600, and attempts to open existing files will fail (the latter is important to prevent symlink attacks). It is the caller's responsibility to close the file. Args: file_path: Path of the file to be created. Returns: A writable file object for the opened file. Raises: CommandException: if an error occurred when opening the file (including when the file already exists). """ flags = os.O_RDWR | os.O_CREAT | os.O_EXCL # Accommodate Windows; copied from python2.6/tempfile.py. if hasattr(os, 'O_NOINHERIT'): flags |= os.O_NOINHERIT try: fd = os.open(file_path, flags, 0o600) except (OSError, IOError) as e: raise CommandException('Failed to open %s for writing: %s' % (file_path, e)) return os.fdopen(fd, 'w') def _CheckPrivateKeyFilePermissions(self, file_path): """Checks that the file has reasonable permissions for a private key. In particular, check that the filename provided by the user is not world- or group-readable. If either of these are true, we issue a warning and offer to fix the permissions. Args: file_path: The name of the private key file. """ if system_util.IS_WINDOWS: # For Windows, this check doesn't work (it actually just checks whether # the file is read-only). Since Windows files have a complicated ACL # system, this check doesn't make much sense on Windows anyway, so we # just don't do it. return st = os.stat(file_path) if bool((stat.S_IRGRP | stat.S_IROTH) & st.st_mode): self.logger.warn( '\nYour private key file is readable by people other than yourself.\n' 'This is a security risk, since anyone with this information can use ' 'your service account.\n') fix_it = input('Would you like gsutil to change the file ' 'permissions for you? (y/N) ') if fix_it in ('y', 'Y'): try: os.chmod(file_path, 0o400) self.logger.info( '\nThe permissions on your file have been successfully ' 'modified.' '\nThe only access allowed is readability by the user ' '(permissions 0400 in chmod).') except Exception as _: # pylint: disable=broad-except self.logger.warn( '\nWe were unable to modify the permissions on your file.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') else: self.logger.info( '\nYou have chosen to allow this file to be readable by others.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') def _WriteConfigLineMaybeCommented(self, config_file, name, value, desc): """Writes proxy name/value pair or comment line to config file. Writes proxy name/value pair if value is not None. Otherwise writes comment line. Args: config_file: File object to which the resulting config file will be written. name: The config variable name. value: The value, or None. desc: Human readable description (for comment). """ if not value: name = '#%s' % name value = '<%s>' % desc config_file.write('%s = %s\n' % (name, value)) def _WriteProxyConfigFileSection(self, config_file): """Writes proxy section of configuration file. Args: config_file: File object to which the resulting config file will be written. """ config = boto.config config_file.write( '# To use a proxy, edit and uncomment the proxy and proxy_port lines.\n' '# If you need a user/password with this proxy, edit and uncomment\n' '# those lines as well. If your organization also disallows DNS\n' '# lookups by client machines, set proxy_rdns to True (the default).\n' '# If you have installed gsutil through the Cloud SDK and have \n' '# configured proxy settings in gcloud, those proxy settings will \n' '# override any other options (including those set here, along with \n' '# any settings in proxy-related environment variables). Otherwise, \n' '# if proxy_host and proxy_port are not specified in this file and\n' '# one of the OS environment variables http_proxy, https_proxy, or\n' '# HTTPS_PROXY is defined, gsutil will use the proxy server specified\n' '# in these environment variables, in order of precedence according\n' '# to how they are listed above.\n') self._WriteConfigLineMaybeCommented(config_file, 'proxy', config.get_value('Boto', 'proxy', None), 'proxy host') self._WriteConfigLineMaybeCommented( config_file, 'proxy_type', config.get_value('Boto', 'proxy_type', None), 'proxy type (socks4, socks5, http) | Defaults to http') self._WriteConfigLineMaybeCommented( config_file, 'proxy_port', config.get_value('Boto', 'proxy_port', None), 'proxy port') self._WriteConfigLineMaybeCommented( config_file, 'proxy_user', config.get_value('Boto', 'proxy_user', None), 'proxy user') self._WriteConfigLineMaybeCommented( config_file, 'proxy_pass', config.get_value('Boto', 'proxy_pass', None), 'proxy password') self._WriteConfigLineMaybeCommented( config_file, 'proxy_rdns', config.get_value('Boto', 'proxy_rdns', False), 'let proxy server perform DNS lookups (True,False); socks proxy not supported' ) # pylint: disable=dangerous-default-value,too-many-statements def _WriteBotoConfigFile(self, config_file, cred_type=CredTypes.OAUTH2_USER_ACCOUNT, configure_auth=True): """Creates a boto config file interactively. Needed credentials are obtained interactively, either by asking the user for access key and secret, or by walking the user through the OAuth2 approval flow. Args: config_file: File object to which the resulting config file will be written. cred_type: There are three options: - for HMAC, ask the user for access key and secret - for OAUTH2_USER_ACCOUNT, raise an error - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service account email address and private key file (and if the file is a .p12 file, the password for that file). configure_auth: Boolean, whether or not to configure authentication in the generated file. """ # Collect credentials provider_map = {'aws': 'aws', 'google': 'gs'} uri_map = {'aws': 's3', 'google': 'gs'} key_ids = {} sec_keys = {} service_account_key_is_json = False if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: gs_service_key_file = input('What is the full path to your private ' 'key file? ') # JSON files have the email address built-in and don't require a # password. try: with open(gs_service_key_file, 'rb') as key_file_fp: json.loads(key_file_fp.read()) service_account_key_is_json = True except ValueError: if not HAS_CRYPTO: raise CommandException( 'Service account authentication via a .p12 file requires ' 'either\nPyOpenSSL or PyCrypto 2.6 or later. Please install ' 'either of these\nto proceed, use a JSON-format key file, or ' 'configure a different type of credentials.') if not service_account_key_is_json: gs_service_client_id = input('What is your service account email ' 'address? ') gs_service_key_file_password = input('\n'.join( textwrap.wrap( 'What is the password for your service key file [if you ' 'haven\'t set one explicitly, leave this line blank]?')) + ' ') self._CheckPrivateKeyFilePermissions(gs_service_key_file) elif cred_type == CredTypes.OAUTH2_USER_ACCOUNT: raise CommandException( 'The user account authentication flow no longer works as of ' 'February 1, 2023. Tokens generated before this date will ' 'continue to work. To authenticate with your user account, ' 'install gsutil via Cloud SDK and run "gcloud auth login"') elif cred_type == CredTypes.HMAC: got_creds = False for provider in provider_map: if provider == 'google': key_ids[provider] = input('What is your %s access key ID? ' % provider) sec_keys[provider] = input('What is your %s secret access ' 'key? ' % provider) got_creds = True if not key_ids[provider] or not sec_keys[provider]: raise CommandException( 'Incomplete credentials provided. Please try again.') if not got_creds: raise CommandException('No credentials provided. Please try again.') # Write the config file prelude. config_file.write(CONFIG_PRELUDE_CONTENT.lstrip()) config_file.write( '# This file was created by gsutil version %s at %s.\n' % (gslib.VERSION, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) config_file.write( '#\n# You can create additional configuration files by ' 'running\n# gsutil config [options] [-o <config-file>]\n\n\n') # Write the config file Credentials section. config_file.write('[Credentials]\n\n') if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: config_file.write('# Google OAuth2 service account credentials ' '(for "gs://" URIs):\n') config_file.write('gs_service_key_file = %s\n' % gs_service_key_file) if not service_account_key_is_json: config_file.write('gs_service_client_id = %s\n' % gs_service_client_id) if not gs_service_key_file_password: config_file.write( '# If you would like to set your password, you can do so\n' '# using the following commands (replaced with your\n' '# information):\n' '# "openssl pkcs12 -in cert1.p12 -out temp_cert.pem"\n' '# "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12"\n' '# "rm -f temp_cert.pem"\n' '# Your initial password is "notasecret" - for more\n' '# information, please see \n' '# http://www.openssl.org/docs/apps/pkcs12.html.\n') config_file.write('#gs_service_key_file_password =\n\n') else: config_file.write('gs_service_key_file_password = %s\n\n' % gs_service_key_file_password) else: config_file.write( '# To add Google OAuth2 credentials ("gs://" URIs), ' 'edit and uncomment the\n# following line:\n' '#gs_oauth2_refresh_token = <your OAuth2 refresh token>\n\n') else: if system_util.InvokedViaCloudSdk(): config_file.write( '# Google OAuth2 credentials are managed by the Cloud SDK and\n' '# do not need to be present in this file.\n') for provider in provider_map: key_prefix = provider_map[provider] uri_scheme = uri_map[provider] if provider in key_ids and provider in sec_keys: config_file.write('# %s credentials ("%s://" URIs):\n' % (provider, uri_scheme)) config_file.write('%s_access_key_id = %s\n' % (key_prefix, key_ids[provider])) config_file.write('%s_secret_access_key = %s\n' % (key_prefix, sec_keys[provider])) else: config_file.write( '# To add HMAC %s credentials for "%s://" URIs, edit and ' 'uncomment the\n# following two lines:\n' '#%s_access_key_id = <your %s access key ID>\n' '#%s_secret_access_key = <your %s secret access key>\n' % (provider, uri_scheme, key_prefix, provider, key_prefix, provider)) host_key = Provider.HostKeyMap[provider] config_file.write( '# The ability to specify an alternate storage host and port\n' '# is primarily for cloud storage service developers.\n' '# Setting a non-default gs_host only works if prefer_api=xml.\n' '#%s_host = <alternate storage host address>\n' '#%s_port = <alternate storage host port>\n' '# In some cases, (e.g. VPC requests) the "host" HTTP header should\n' '# be different than the host used in the request URL.\n' '#%s_host_header = <alternate storage host header>\n' % (host_key, host_key, host_key)) if host_key == 'gs': config_file.write( '#%s_json_host = <alternate JSON API storage host address>\n' '#%s_json_port = <alternate JSON API storage host port>\n' '#%s_json_host_header = <alternate JSON API storage host header>\n\n' % (host_key, host_key, host_key)) config_file.write( '# To impersonate a service account for "%s://" URIs over\n' '# JSON API, edit and uncomment the following line:\n' '#%s_impersonate_service_account = <service account email>\n\n') # Add device certificate mTLS fields. config_file.write( textwrap.dedent("""\ # This configuration setting enables or disables mutual TLS # authentication. The default value for this setting is "false". When # set to "true", gsutil uses the configured client certificate as # transport credential to access the APIs. The use of mTLS ensures that # the access originates from a trusted enterprise device. When enabled, # the client certificate is auto discovered using the endpoint # verification agent. When set to "true" but no client certificate or # key is found, users receive an error. #use_client_certificate = False # The command line to execute, which prints the # certificate, private key, or password to use in # conjunction with "use_client_certificate = True". #cert_provider_command = <Absolute path to command to run for # certification. Ex: "/scripts/gen_cert.sh"> """)) # Write the config file Boto section. config_file.write('%s\n' % CONFIG_BOTO_SECTION_CONTENT) self._WriteProxyConfigFileSection(config_file) # Write the Google Compute Engine section. config_file.write(CONFIG_GOOGLECOMPUTE_SECTION_CONTENT) # Write the config file GSUtil section that doesn't depend on user input. config_file.write(CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT) # Write the default API version. config_file.write(""" # 'default_api_version' specifies the default Google Cloud Storage XML API # version to use. If not set below gsutil defaults to API version 1. """) api_version = 2 if cred_type == CredTypes.HMAC: api_version = 1 config_file.write('default_api_version = %d\n' % api_version) # Write the config file GSUtil section that includes the default # project ID input from the user. if not system_util.InvokedViaCloudSdk(): default_project_id = input('What is your project-id? ').strip() project_id_section_prelude = """ # 'default_project_id' specifies the default Google Cloud Storage project ID to # use with the 'mb' and 'ls' commands. This default can be overridden by # specifying the -p option to the 'mb' and 'ls' commands. """ if not default_project_id: raise CommandException( 'No default project ID entered. The default project ID is needed ' 'by the\nls and mb commands; please try again.') config_file.write('%sdefault_project_id = %s\n\n\n' % (project_id_section_prelude, default_project_id)) CheckAndMaybePromptForAnalyticsEnabling() # Write the config file OAuth2 section that doesn't depend on user input. config_file.write(CONFIG_OAUTH2_CONFIG_CONTENT) config_file.write('#client_id = <OAuth2 client id>\n' '#client_secret = <OAuth2 client secret>\n') def RunCommand(self): """Command entry point for the config command.""" cred_type = CredTypes.OAUTH2_USER_ACCOUNT output_file_name = None has_a = False has_e = False configure_auth = True for opt, opt_arg in self.sub_opts: if opt == '-a': cred_type = CredTypes.HMAC has_a = True elif opt == '-e': cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT has_e = True elif opt == '-n': configure_auth = False elif opt == '-o': output_file_name = opt_arg else: self.RaiseInvalidArgumentException() if has_e and has_a: raise CommandException('Both -a and -e cannot be specified. Please see ' '"gsutil help config" for more information.') if not configure_auth and (has_a or has_e): raise CommandException( 'The -a and -e flags cannot be specified with the -n flag. Please ' 'see "gsutil help config" for more information.') # Don't allow users to configure Oauth2 (any option other than -a and -n) # when running in the Cloud SDK, unless they have the Cloud SDK configured # not to pass credentials to gsutil. if (system_util.InvokedViaCloudSdk() and system_util.CloudSdkCredPassingEnabled() and not has_a and configure_auth): raise CommandException('\n'.join([ 'OAuth2 is the preferred authentication mechanism with the Cloud ' 'SDK.', 'Run "gcloud auth login" to configure authentication, unless:', '\n'.join( textwrap.wrap( 'You don\'t want gsutil to use OAuth2 credentials from the Cloud ' 'SDK, but instead want to manage credentials with .boto files ' 'generated by running "gsutil config"; in which case run ' '"gcloud config set pass_credentials_to_gsutil false".', initial_indent='- ', subsequent_indent=' ')), '\n'.join( textwrap.wrap( 'You want to authenticate with an HMAC access key and secret, in ' 'which case run "gsutil config -a".', initial_indent='- ', subsequent_indent=' ')), ])) if system_util.InvokedViaCloudSdk() and has_a: sys.stderr.write('\n'.join( textwrap.wrap( 'This command will configure HMAC credentials, but gsutil will use ' 'OAuth2 credentials from the Cloud SDK by default. To make sure ' 'the HMAC credentials are used, run: "gcloud config set ' 'pass_credentials_to_gsutil false".')) + '\n\n') default_config_path_bak = None if not output_file_name: # Check to see if a default config file name is requested via # environment variable. If so, use it, otherwise use the hard-coded # default file. Then use the default config file name, if it doesn't # exist or can be moved out of the way without clobbering an existing # backup file. boto_config_from_env = os.environ.get('BOTO_CONFIG', None) if boto_config_from_env: default_config_path = boto_config_from_env else: default_config_path = os.path.expanduser(os.path.join('~', '.boto')) if not os.path.exists(default_config_path): output_file_name = default_config_path else: default_config_path_bak = default_config_path + '.bak' if os.path.exists(default_config_path_bak): raise CommandException('Cannot back up existing config ' 'file "%s": backup file exists ("%s").' % (default_config_path, default_config_path_bak)) else: try: sys.stderr.write( 'Backing up existing config file "%s" to "%s"...\n' % (default_config_path, default_config_path_bak)) os.rename(default_config_path, default_config_path_bak) except Exception as e: raise CommandException( 'Failed to back up existing config ' 'file ("%s" -> "%s"): %s.' % (default_config_path, default_config_path_bak, e)) output_file_name = default_config_path if output_file_name == '-': output_file = sys.stdout else: output_file = self._OpenConfigFile(output_file_name) sys.stderr.write('\n'.join( textwrap.wrap( 'This command will create a boto config file at %s containing your ' 'credentials, based on your responses to the following questions.' % output_file_name)) + '\n') # Catch ^C so we can restore the backup. RegisterSignalHandler(signal.SIGINT, _CleanupHandler) try: self._WriteBotoConfigFile(output_file, cred_type=cred_type, configure_auth=configure_auth) except Exception as e: user_aborted = isinstance(e, AbortException) if user_aborted: sys.stderr.write('\nCaught ^C; cleaning up\n') # If an error occurred during config file creation, remove the invalid # config file and restore the backup file. if output_file_name != '-': output_file.close() os.unlink(output_file_name) try: if default_config_path_bak: sys.stderr.write('Restoring previous backed up file (%s)\n' % default_config_path_bak) os.rename(default_config_path_bak, output_file_name) except Exception as e: # Raise the original exception so that we can see what actually went # wrong, rather than just finding out that we died before assigning # a value to default_config_path_bak. raise e raise if output_file_name != '-': output_file.close() if not boto.config.has_option('Boto', 'proxy'): sys.stderr.write('\n' + '\n'.join( textwrap.wrap( 'Boto config file "%s" created.\nIf you need to use a proxy to ' 'access the Internet please see the instructions in that file.' % output_file_name)) + '\n') return 0 def _CleanupHandler(unused_signalnum, unused_handler): raise AbortException('User interrupted config command') rewrite.py 0000644 00000063314 15025166704 0006614 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rewrite command (in-place cloud object transformation).""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import textwrap import time from apitools.base.py import encoding from boto import config from gslib.cloud_api import EncryptionException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.progress_callback import FileProgressCallbackHandler from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.encryption_helper import MAX_DECRYPTION_KEYS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.text_util import ConvertRecursiveToFlatWildcard from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import text_util from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_PROGRESS_INDICATOR_COLUMNS = 65 _SYNOPSIS = """ gsutil rewrite -k [-O] [-f] [-r] [-s] url... gsutil rewrite -k [-O] [-f] [-r] [-s] -I gsutil rewrite -s [-k] [-O] [-f] [-r] url... gsutil rewrite -s [-k] [-O] [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rewrite command rewrites cloud objects, applying the specified transformations to them. The transformation(s) are atomic for each affected object and applied based on the input transformation flags. Object metadata values are preserved unless altered by a transformation. At least one transformation flag, -k or -s, must be included in the command. The -k flag is supported to add, rotate, or remove encryption keys on objects. For example, the command: gsutil rewrite -k -r gs://bucket updates all objects in gs://bucket with the current encryption key from your boto config file, which may either be a base64-encoded CSEK or the fully-qualified name of a Cloud KMS key. The rewrite command acts only on live object versions, so specifying a URL with a generation number fails. If you want to rewrite a noncurrent version, first copy it to the live version, then rewrite it, for example: gsutil cp gs://bucket/object#123 gs://bucket/object gsutil rewrite -k gs://bucket/object You can use the -s option to specify a new storage class for objects. For example, the command: gsutil rewrite -s nearline gs://bucket/foo rewrites the object, changing its storage class to nearline. If you specify the -k option and you have an encryption key set in your boto configuration file, the rewrite command skips objects that are already encrypted with the specified key. For example, if you run: gsutil rewrite -k -r gs://bucket and gs://bucket contains objects encrypted with the key specified in your boto configuration file, gsutil skips rewriting those objects and only rewrites objects that are not encrypted with the specified key. This avoids the cost of performing redundant rewrite operations. If you specify the -k option and you do not have an encryption key set in your boto configuration file, gsutil always rewrites each object, without explicitly specifying an encryption key. This results in rewritten objects being encrypted with either the bucket's default KMS key (if one is set) or Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to determine whether the operation is redundant (and thus skippable) because gsutil cannot be sure how the object is encrypted after the rewrite. Note that if your goal is to encrypt objects with a bucket's default KMS key, you can avoid redundant rewrite costs by specifying the bucket's default KMS key in your boto configuration file; this allows gsutil to perform an accurate comparison of the objects' current and desired encryption configurations and skip rewrites for objects already encrypted with that key. If have an encryption key set in your boto configuration file and specify multiple transformations, gsutil only skips those that would not change the object's state. For example, if you run: gsutil rewrite -s nearline -k -r gs://bucket and gs://bucket contains objects that already match the encryption configuration but have a storage class of standard, the only transformation applied to those objects would be the change in storage class. You can pass a list of URLs (one per line) to rewrite on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to rewrite objects identified by a program, such as: some_program | gsutil -m rewrite -k -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. The rewrite command requires OWNER permissions on each object to preserve object ACLs. You can bypass this by using the -O flag, which causes gsutil not to read the object's ACL and instead apply the default object ACL to the rewritten object: gsutil rewrite -k -O -r gs://bucket <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when rewriting multiple objects. If some of the objects could not be rewritten, gsutil's exit status is non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rewrite ...". -I Causes gsutil to read the list of objects to rewrite from stdin. This allows you to run a program that generates the list of objects to rewrite. -k Rewrite objects with the current encryption key specified in your boto configuration file. The value for encryption_key may be either a base64-encoded CSEK or a fully-qualified KMS key name. If no value is specified for encryption_key, gsutil ignores this flag. Instead, rewritten objects are encrypted with the bucket's default KMS key, if one is set, or Google-managed encryption, if no default KMS key is set. -O When a bucket has uniform bucket-level access (UBLA) enabled, the -O flag is required and skips all ACL checks. When a bucket has UBLA disabled, the -O flag rewrites objects with the bucket's default object ACL instead of the existing object ACL. This is needed if you do not have OWNER permission on the object. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents to be rewritten recursively. -s <class> Rewrite objects using the specified storage class. """) def _RewriteExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) cls.op_failure_count += 1 def _RewriteFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RewriteFunc(name_expansion_result, thread_state=thread_state) def GenerationCheckGenerator(url_strs): """Generator function that ensures generation-less (live) arguments.""" for url_str in url_strs: if StorageUrlFromString(url_str).generation is not None: raise CommandException('"rewrite" called on URL with generation (%s).' % url_str) yield url_str class _TransformTypes(object): """Enum class for valid transforms.""" CRYPTO_KEY = 'crypto_key' STORAGE_CLASS = 'storage_class' class RewriteCommand(Command): """Implementation of gsutil rewrite command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rewrite', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='fkIrROs:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rewrite', help_name_aliases=['rekey', 'rotate'], help_type='command_help', help_one_line_summary='Rewrite objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'update'], flag_map={ '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), # Adding encryptions handled in shim_util.py. '-k': None if config.get('GSUtil', 'encryption_key', None) else GcloudStorageFlag('--clear-encryption-key'), '-O': GcloudStorageFlag('--no-preserve-acl'), '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-s': GcloudStorageFlag('-s'), }, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"rewrite" called on URL with unsupported provider: %s' % str(url)) def RunCommand(self): """Command entry point for the rewrite command.""" self.continue_on_error = self.parallel_operations self.csek_hash_to_keywrapper = {} self.dest_storage_class = None self.no_preserve_acl = False self.read_args_from_stdin = False self.supported_transformation_flags = ['-k', '-s'] self.transform_types = set() self.op_failure_count = 0 self.boto_file_encryption_keywrapper = GetEncryptionKeyWrapper(config) self.boto_file_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256 if self.boto_file_encryption_keywrapper else None) if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-k': self.transform_types.add(_TransformTypes.CRYPTO_KEY) elif o == '-I': self.read_args_from_stdin = True elif o == '-O': self.no_preserve_acl = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True elif o == '-s': self.transform_types.add(_TransformTypes.STORAGE_CLASS) self.dest_storage_class = NormalizeStorageClass(a) if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rewrite command (without -I) expects at ' 'least one URL.') url_strs = self.args if not self.transform_types: raise CommandException( 'rewrite command requires at least one transformation flag. ' 'Currently supported transformation flags: %s' % self.supported_transformation_flags) self.preconditions = PreconditionsFromHeaders(self.headers or {}) url_strs_generator = GenerationCheckGenerator(url_strs) # Convert recursive flag to flat wildcard to avoid performing multiple # listings. if self.recursion_requested: url_strs_generator = ConvertRecursiveToFlatWildcard(url_strs_generator) # Expand the source argument(s). name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs_generator, self.recursion_requested, project_id=self.project_id, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name', 'size']) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: # Perform the same recursive-to-flat conversion on original url_strs so # that it is as true to the original iterator as possible. seek_ahead_url_strs = ConvertRecursiveToFlatWildcard(url_strs) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), seek_ahead_url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Rather than have each worker repeatedly calculate the sha256 hash for each # decryption_key in the boto config, do this once now and cache the results. for i in range(0, MAX_DECRYPTION_KEYS): key_number = i + 1 keywrapper = CryptoKeyWrapperFromKey( config.get('GSUtil', 'decryption_key%s' % str(key_number), None)) if keywrapper is None: # Stop at first attribute absence in lexicographical iteration. break if keywrapper.crypto_type == CryptoKeyType.CSEK: self.csek_hash_to_keywrapper[keywrapper.crypto_key_sha256] = keywrapper # Also include the encryption_key, since it should be used to decrypt and # then encrypt if the object's CSEK should remain the same. if self.boto_file_encryption_sha256 is not None: self.csek_hash_to_keywrapper[self.boto_file_encryption_sha256] = ( self.boto_file_encryption_keywrapper) if self.boto_file_encryption_keywrapper is None: msg = '\n'.join( textwrap.wrap( 'NOTE: No encryption_key was specified in the boto configuration ' 'file, so gsutil will not provide an encryption key in its rewrite ' 'API requests. This will decrypt the objects unless they are in ' 'buckets with a default KMS key set, in which case the service ' 'will automatically encrypt the rewritten objects with that key.') ) print('%s\n' % msg, file=sys.stderr) # Perform rewrite requests in parallel (-m) mode, if requested. self.Apply(_RewriteFuncWrapper, name_expansion_iterator, _RewriteExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count'], seek_ahead_iterator=seek_ahead_iterator) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be rewritten.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RewriteFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) transform_url = name_expansion_result.expanded_storage_url self.CheckProvider(transform_url) # Get all fields so that we can ensure that the target metadata is # specified correctly. src_metadata = gsutil_api.GetObjectMetadata( transform_url.bucket_name, transform_url.object_name, generation=transform_url.generation, provider=transform_url.scheme) if self.no_preserve_acl: # Leave ACL unchanged. src_metadata.acl = [] elif not src_metadata.acl: raise CommandException( 'No OWNER permission found for object %s. If your bucket has uniform ' 'bucket-level access (UBLA) enabled, include the -O option in your ' 'command to avoid this error. If your bucket does not use UBLA, you ' 'can use the -O option to apply the bucket\'s default object ACL ' 'when rewriting.' % transform_url) # Note: If other transform types are added, they must ensure that the # encryption key configuration matches the boto configuration, because # gsutil maintains an invariant that all objects it writes use the # encryption_key value (including decrypting if no key is present). # Store metadata about src encryption to make logic below easier to read. src_encryption_kms_key = (src_metadata.kmsKeyName if src_metadata.kmsKeyName else None) src_encryption_sha256 = None if (src_metadata.customerEncryption and src_metadata.customerEncryption.keySha256): src_encryption_sha256 = src_metadata.customerEncryption.keySha256 # In python3, hashes are bytes, use ascii since it should be ascii src_encryption_sha256 = src_encryption_sha256.encode('ascii') src_was_encrypted = (src_encryption_sha256 is not None or src_encryption_kms_key is not None) # Also store metadata about dest encryption. dest_encryption_kms_key = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CMEK): dest_encryption_kms_key = self.boto_file_encryption_keywrapper.crypto_key dest_encryption_sha256 = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CSEK): dest_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256) should_encrypt_dest = self.boto_file_encryption_keywrapper is not None encryption_unchanged = (src_encryption_sha256 == dest_encryption_sha256 and src_encryption_kms_key == dest_encryption_kms_key) # Prevent accidental key rotation. if (_TransformTypes.CRYPTO_KEY not in self.transform_types and not encryption_unchanged): raise EncryptionException( 'The "-k" flag was not passed to the rewrite command, but the ' 'encryption_key value in your boto config file did not match the key ' 'used to encrypt the object "%s" (hash: %s). To encrypt the object ' 'using a different key, you must specify the "-k" flag.' % (transform_url, src_encryption_sha256)) # Determine if we can skip this rewrite operation (this should only be done # when ALL of the specified transformations are redundant). redundant_transforms = [] # STORAGE_CLASS transform is redundant if the target storage class matches # the existing storage class. if (_TransformTypes.STORAGE_CLASS in self.transform_types and self.dest_storage_class == NormalizeStorageClass( src_metadata.storageClass)): redundant_transforms.append('storage class') # CRYPTO_KEY transform is redundant if we're using the same encryption # key that was used to encrypt the source. However, if no encryption key was # specified, we should still perform the rewrite. This results in the # rewritten object either being encrypted with its bucket's default KMS key # or having no CSEK/CMEK encryption applied. While we could attempt fetching # the bucket's metadata and checking its default KMS key before performing # the rewrite (in the case where we appear to be transitioning from # no key to no key), that is vulnerable to the race condition where the # default KMS key is changed between when we check it and when we rewrite # the object. if (_TransformTypes.CRYPTO_KEY in self.transform_types and should_encrypt_dest and encryption_unchanged): redundant_transforms.append('encryption key') if len(redundant_transforms) == len(self.transform_types): self.logger.info('Skipping %s, all transformations were redundant: %s' % (transform_url, redundant_transforms)) return # First make a deep copy of the source metadata, then overwrite any # requested attributes (e.g. if a storage class change was specified). dest_metadata = encoding.PyValueToMessage( apitools_messages.Object, encoding.MessageToPyValue(src_metadata)) # Remove some unnecessary/invalid fields. dest_metadata.generation = None # Service has problems if we supply an ID, but it is responsible for # generating one, so it is not necessary to include it here. dest_metadata.id = None # Ensure we don't copy over the KMS key name or CSEK key info from the # source object; those should only come from the boto config's # encryption_key value. dest_metadata.customerEncryption = None dest_metadata.kmsKeyName = None # Both a storage class change and CMEK encryption should be set as part of # the dest object's metadata. CSEK encryption, if specified, is added to the # request later via headers obtained from the keywrapper value passed to # encryption_tuple. if _TransformTypes.STORAGE_CLASS in self.transform_types: dest_metadata.storageClass = self.dest_storage_class if dest_encryption_kms_key is not None: dest_metadata.kmsKeyName = dest_encryption_kms_key # Make sure we have the CSEK key necessary to decrypt. decryption_keywrapper = None if src_encryption_sha256 is not None: if src_encryption_sha256 in self.csek_hash_to_keywrapper: decryption_keywrapper = ( self.csek_hash_to_keywrapper[src_encryption_sha256]) else: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s' % (src_encryption_sha256, transform_url)) operation_name = 'Rewriting' if _TransformTypes.CRYPTO_KEY in self.transform_types: if src_was_encrypted and should_encrypt_dest: if not encryption_unchanged: operation_name = 'Rotating' # Else, keep "Rewriting". This might occur when -k was specified and was # redundant, but we're performing the operation anyway because some # other transformation was not redundant. elif src_was_encrypted and not should_encrypt_dest: operation_name = 'Decrypting' elif not src_was_encrypted and should_encrypt_dest: operation_name = 'Encrypting' # TODO: Remove this call (used to verify tests) and make it processed by # the UIThread. sys.stderr.write( _ConstructAnnounceText(operation_name, transform_url.url_string)) sys.stderr.flush() # Message indicating beginning of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=False, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=transform_url, operation_name=operation_name).call gsutil_api.CopyObject(src_metadata, dest_metadata, src_generation=transform_url.generation, preconditions=self.preconditions, progress_callback=progress_callback, decryption_tuple=decryption_keywrapper, encryption_tuple=self.boto_file_encryption_keywrapper, provider=transform_url.scheme, fields=[]) # Message indicating end of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=True, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) def _ConstructAnnounceText(operation_name, url_string): """Constructs announce text for ongoing operations on url_string. This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and informs the rewrite-related operation ('Encrypting', 'Rotating', or 'Decrypting'). Args: operation_name: String describing the operation, i.e. 'Rotating' or 'Encrypting'. url_string: String describing the file/object being processed. Returns: Formatted announce text for outputting operation progress. """ # Operation name occupies 10 characters (enough for 'Encrypting'), plus a # space. The rest is used for url_string. If a longer operation name is # used, it will be truncated. We can revisit this size if we need to support # a longer operation, but want to make sure the terminal output is meaningful. justified_op_string = operation_name[:10].ljust(11) start_len = len(justified_op_string) end_len = len(': ') if (start_len + len(url_string) + end_len > MAX_PROGRESS_INDICATOR_COLUMNS): ellipsis_len = len('...') url_string = '...%s' % url_string[-(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):] base_announce_text = '%s%s:' % (justified_op_string, url_string) format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS return format_str.format(base_announce_text) du.py 0000644 00000024753 15025166704 0005547 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like du command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import locale import sys import six from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import ls_helper from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import print_to_fd from gslib.utils.unit_util import MakeHumanReadable from gslib.utils import text_util _SYNOPSIS = """ gsutil du url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The du command displays the amount of space in bytes used up by the objects in a bucket, subdirectory, or project. The syntax emulates the Linux ``du -b`` command, which reports the disk usage of files and subdirectories. For example, the following command reports the total space used by all objects and subdirectories under gs://your-bucket/dir: gsutil du -s -a gs://your-bucket/dir <B>OPTIONS</B> -0 Ends each output line with a 0 byte rather than a newline. You can use this to make the output machine-readable. -a Includes both live and noncurrent object versions. Also prints the generation and metageneration number for each listed object. If this flag is not specified, only live object versions are included. -c Includes a total size at the end of the output. -e Exclude a pattern from the report. Example: -e "*.o" excludes any object that ends in ".o". Can be specified multiple times. -h Prints object sizes in human-readable format. For example, ``1 KiB``, ``234 MiB``, or ``2GiB``. -s Displays only the total size for each argument, omitting the list of individual objects. -X Similar to ``-e``, but excludes patterns from the given file. The patterns to exclude should be listed one per line. <B>EXAMPLES</B> To list the size of each object in a bucket: gsutil du gs://bucketname To list the size of each object in the ``prefix`` subdirectory: gsutil du gs://bucketname/prefix/* To include the total number of bytes in human-readable form: gsutil du -ch gs://bucketname To see only the summary of the total number of (live) bytes in two given buckets: gsutil du -s gs://bucket1 gs://bucket2 To list the size of each object in a bucket with `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ enabled, including noncurrent objects: gsutil du -a gs://bucketname To list the size of each object in a bucket, except objects that end in ".bak", with each object printed ending in a null byte: gsutil du -e "*.bak" -0 gs://bucketname To list the size of each bucket in a project and the total size of the project: gsutil -o GSUtil:default_project_id=project-name du -shc """) class DuCommand(Command): """Implementation of gsutil du command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'du', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='0ace:hsX:', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='du', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object size usage', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'du'], flag_map={ '-0': GcloudStorageFlag('--zero-terminator'), '-a': GcloudStorageFlag('--all-versions'), '-c': GcloudStorageFlag('--total'), '-e': GcloudStorageFlag('--exclude-name-pattern'), '-h': GcloudStorageFlag('--readable-sizes'), '-s': GcloudStorageFlag('--summarize'), '-X': GcloudStorageFlag('--exclude-name-pattern-file'), }, ) def _PrintSummaryLine(self, num_bytes, name): size_string = (MakeHumanReadable(num_bytes) if self.human_readable else six.text_type(num_bytes)) text_util.print_to_fd('{size:<11} {name}'.format( size=size_string, name=six.ensure_text(name)), end=self.line_ending) def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref): """Print listing info for given bucket_listing_ref. Args: bucket_listing_ref: BucketListing being listed. Returns: Tuple (number of objects, object size) Raises: Exception: if calling bug encountered. """ obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 if not self.summary_only: url_detail = '{size:<11} {url}{ending}'.format( size=size_string, url=six.ensure_text(url_str), ending=six.ensure_text(self.line_ending)) print_to_fd(url_detail, file=sys.stdout, end='') return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the du command.""" self.line_ending = '\n' self.all_versions = False self.produce_total = False self.human_readable = False self.summary_only = False self.exclude_patterns = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-0': self.line_ending = '\0' elif o == '-a': self.all_versions = True elif o == '-c': self.produce_total = True elif o == '-e': self.exclude_patterns.append(a) elif o == '-h': self.human_readable = True elif o == '-s': self.summary_only = True elif o == '-X': if a == '-': f = sys.stdin f_close = False else: f = open(a, 'r') if six.PY2 else open(a, 'r', encoding=UTF8) f_close = True self.exclude_patterns = [six.ensure_text(line.strip()) for line in f] if f_close: f.close() if not self.args: # Default to listing all gs buckets. self.args = ['gs://'] total_bytes = 0 got_nomatch_errors = False def _PrintObjectLong(blr): return self._PrintInfoAboutBucketListingRef(blr) def _PrintNothing(unused_blr=None): pass def _PrintDirectory(num_bytes, blr): if not self.summary_only: self._PrintSummaryLine(num_bytes, blr.url_string.encode(UTF8)) for url_arg in self.args: top_level_storage_url = StorageUrlFromString(url_arg) if top_level_storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_listing_fields = ['size'] listing_helper = ls_helper.LsHelper( self.WildcardIterator, self.logger, print_object_func=_PrintObjectLong, print_dir_func=_PrintNothing, print_dir_header_func=_PrintNothing, print_dir_summary_func=_PrintDirectory, print_newline_func=_PrintNothing, all_versions=self.all_versions, should_recurse=True, exclude_patterns=self.exclude_patterns, fields=bucket_listing_fields) # LsHelper expands to objects and prefixes, so perform a top-level # expansion first. if top_level_storage_url.IsProvider(): # Provider URL: use bucket wildcard to iterate over all buckets. top_level_iter = self.WildcardIterator( '%s://*' % top_level_storage_url.scheme).IterBuckets(bucket_fields=['id']) elif top_level_storage_url.IsBucket(): top_level_iter = self.WildcardIterator( '%s://%s' % (top_level_storage_url.scheme, top_level_storage_url.bucket_name)).IterBuckets( bucket_fields=['id']) else: top_level_iter = [BucketListingObject(top_level_storage_url)] for blr in top_level_iter: storage_url = blr.storage_url if storage_url.IsBucket() and self.summary_only: storage_url = StorageUrlFromString( storage_url.CreatePrefixUrl(wildcard_suffix='**')) _, exp_objs, exp_bytes = listing_helper.ExpandUrlAndPrint(storage_url) if (storage_url.IsObject() and exp_objs == 0 and ContainsWildcard(url_arg) and not self.exclude_patterns): got_nomatch_errors = True total_bytes += exp_bytes if self.summary_only: self._PrintSummaryLine(exp_bytes, blr.url_string.rstrip('/').encode(UTF8)) if self.produce_total: self._PrintSummaryLine(total_bytes, 'total') if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') return 0 web.py 0000644 00000023570 15025166704 0005710 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of website configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil web set [-m <main_page_suffix>] [-e <error_page>] gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil web get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``gsutil web set`` command allows you to configure or disable the website configuration on your buckets. The ``set`` sub-command has the following options (omit both options to unset the configuration): <B>SET OPTIONS</B> -m <index.html> Specifies the object name to serve when a bucket listing is requested via a custom domain. -e <404.html> Specifies the error page to serve when a request is made for a non-existent object via a custom domain """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil web get`` command retrieves the web semantics configuration for a bucket and displays a JSON representation of the configuration. In Google Cloud Storage, this would look like the following: { "notFoundPage": "404.html", "mainPageSuffix": "index.html" } """ _DESCRIPTION = """ Cloud Storage allows you to configure a bucket to behave like a static website when the bucket is accessed through a `custom domain <https://cloud.google.com/storage/docs/request-endpoints#custom-domains>`_. For example, if you set a ``main_page_suffix``, a subsequent GET bucket request through a custom domain serves the specified "main" page instead of performing the usual bucket listing. Similarly, if you set an ``error_page``, a subsequent GET object request through a custom domain for a non-existent object serves the specified error page instead of the standard Cloud Storage error. Requests to a bucket through other `endpoints <https://cloud.google.com/storage/docs/request-endpoints>`_ are unaffected by the bucket's website configuration. See `Static website examples and tips <https://cloud.google.com/storage/docs/static-website>`_ for additional examples and information. Notes: 1. Because the main page is only served when a bucket listing request is made via a custom domain endpoint, you can continue to use ``gsutil ls`` as you normally would for bucket listing. 2. The main_page_suffix applies to each subdirectory of the bucket. For example, with the main_page_suffix configured to be index.html, a GET request for ``http://www.example.com`` retrieves ``http://www.example.com/index.html`` but shows ``http://www.example.com`` as the URL to the requester, and a GET request for ``http://www.example.com/photos`` retrieves ``http://www.example.com/photos/index.html`` but shows ``http://www.example.com/photos`` as the URL to the requester. 3. There is just one 404.html page. For example, a GET request for ``http://www.example.com/photos/missing`` retrieves ``http://www.example.com/404.html``, not ``http://www.example.com/photos/404.html``. The web command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class WebCommand(Command): """Implementation of gsutil web command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'web', command_name_aliases=['setwebcfg', 'getwebcfg'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='m:e:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='web', help_name_aliases=['getwebcfg', 'setwebcfg'], help_type='command_help', help_one_line_summary=('Set a website configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=website_config,empty=\' has ' 'no website configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, supports_output_translation=True, ), # "set" subcommand handled in get_gcloud_storage_args. }, flag_map={}, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': set_command_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-e': GcloudStorageFlag('--web-error-page'), '-m': GcloudStorageFlag('--web-main-page-suffix'), }) }, flag_map={}, ) if not ('-e' in self.args or '-m' in self.args): set_command_map.gcloud_command['set'].gcloud_command += [ '--clear-web-error-page', '--clear-web-main-page-suffix' ] gcloud_storage_map = set_command_map else: gcloud_storage_map = WebCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GetWeb(self): """Gets website configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['website']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetWebsite(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.website and (bucket_metadata.website.mainPageSuffix or bucket_metadata.website.notFoundPage): sys.stdout.write( str(encoding.MessageToJson(bucket_metadata.website)) + '\n') else: sys.stdout.write('%s has no website configuration.\n' % bucket_url) return 0 def _SetWeb(self): """Sets website configuration for a bucket.""" main_page_suffix = None error_page = None if self.sub_opts: for o, a in self.sub_opts: if o == '-m': main_page_suffix = a elif o == '-e': error_page = a url_args = self.args website = apitools_messages.Bucket.WebsiteValue( mainPageSuffix=main_page_suffix, notFoundPage=error_page) # Iterate over URLs, expanding wildcards and setting the website # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting website configuration on %s...', blr) bucket_metadata = apitools_messages.Bucket(website=website) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the web command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'get': func = self._GetWeb elif action_subcommand == 'set': func = self._SetWeb else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help web".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() versioning.py 0000644 00000020535 15025166704 0007314 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of versioning configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil versioning set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil versioning get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable versioning for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the versioning configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Versioning Configuration <https://cloud.google.com/storage/docs/object-versioning>`_ feature enables you to configure a Google Cloud Storage bucket to keep old versions of objects. The gsutil versioning command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=""](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://").sub("$", ": "),' + 'versioning.enabled' + '.yesno("Enabled", "Suspended"))') _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class VersioningCommand(Command): """Implementation of gsutil versioning command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'versioning', command_name_aliases=['setversioning', 'getversioning'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='versioning', help_name_aliases=[ 'getversioning', 'setversioning', ], help_type='command_help', help_one_line_summary=( 'Enable or suspend versioning for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', '--raw', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--versioning', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-versioning', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetVersioning(self): """Gets versioning configuration for a bucket.""" versioning_arg = self.args[0].lower() if versioning_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( versioning=apitools_messages.Bucket.VersioningValue()) if versioning_arg == 'on': self.logger.info('Enabling versioning for %s...', url) bucket_metadata.versioning.enabled = True else: self.logger.info('Suspending versioning for %s...', url) bucket_metadata.versioning.enabled = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetVersioning(self): """Gets versioning configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['versioning']) for blr in bucket_iter: some_matched = True if blr.root_object.versioning and blr.root_object.versioning.enabled: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Suspended' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the versioning command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetVersioning metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetVersioning versioning_arg = self.args[0].lower() if versioning_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, versioning_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0 __init__.py 0000644 00000076606 15025166704 0006702 0 ustar 00 # Copyright 2006 Duke University # Copyright (C) 2012-2016 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Written by Seth Vidal """ Classes for subcommands of the yum command line interface. """ from __future__ import print_function from __future__ import unicode_literals from dnf.cli.option_parser import OptionParser from dnf.i18n import _ import dnf.cli import dnf.exceptions import dnf.pycomp import dnf.util import logging import os logger = logging.getLogger('dnf') _RPM_VERIFY = _("To diagnose the problem, try running: '%s'.") % \ 'rpm -Va --nofiles --nodigest' _RPM_REBUILDDB = _("You probably have corrupted RPMDB, running '%s'" " might fix the issue.") % 'rpm --rebuilddb' gpg_msg = \ _("""You have enabled checking of packages via GPG keys. This is a good thing. However, you do not have any GPG public keys installed. You need to download the keys for packages you wish to install and install them. You can do that by running the command: rpm --import public.gpg.key Alternatively you can specify the url to the key you would like to use for a repository in the 'gpgkey' option in a repository section and {prog} will install it for you. For more information contact your distribution or package provider.""") def _checkGPGKey(base, cli): """Verify that there are gpg keys for the enabled repositories in the rpm database. :param base: a :class:`dnf.Base` object. :raises: :class:`cli.CliError` """ if not base.conf.gpgcheck: return if not base._gpg_key_check(): for repo in base.repos.iter_enabled(): if (repo.gpgcheck or repo.repo_gpgcheck) and not repo.gpgkey: logger.critical("\n%s\n", gpg_msg.format(prog=dnf.util.MAIN_PROG_UPPER)) logger.critical(_("Problem repository: %s"), repo) raise dnf.cli.CliError def _checkEnabledRepo(base, possible_local_files=()): """Verify that there is at least one enabled repo. :param base: a :class:`dnf.Base` object. :param possible_local_files: the list of strings that could be a local rpms :raises: :class:`cli.CliError`: """ if base.repos._any_enabled(): return for lfile in possible_local_files: if lfile.endswith(".rpm") and os.path.exists(lfile): return scheme = dnf.pycomp.urlparse.urlparse(lfile)[0] if scheme in ('http', 'ftp', 'file', 'https'): return msg = _('There are no enabled repositories in "{}".').format('", "'.join(base.conf.reposdir)) raise dnf.cli.CliError(msg) class Command(object): """Abstract base class for CLI commands.""" aliases = [] # :api summary = "" # :api opts = None def __init__(self, cli): # :api self.cli = cli @property def base(self): # :api return self.cli.base @property def _basecmd(self): return self.aliases[0] @property def output(self): return self.cli.base.output def set_argparser(self, parser): """Define command specific options and arguments. #:api""" pass def pre_configure(self): # :api """Do any command-specific pre-configuration.""" pass def configure(self): # :api """Do any command-specific configuration.""" pass def get_error_output(self, error): """Get suggestions for resolving the given error.""" if isinstance(error, dnf.exceptions.TransactionCheckError): return (_RPM_VERIFY, _RPM_REBUILDDB) raise NotImplementedError('error not supported yet: %s' % error) def run(self): # :api """Execute the command.""" pass def run_resolved(self): """Finalize operation after resolvement""" pass def run_transaction(self): """Finalize operations post-transaction.""" pass class InfoCommand(Command): """A class containing methods needed by the cli to execute the info command. """ aliases = ('info', 'if') summary = _('display details about a package or group of packages') DEFAULT_PKGNARROW = 'all' pkgnarrows = {'available', 'installed', 'extras', 'updates', 'upgrades', 'autoremove', 'recent', 'obsoletes', DEFAULT_PKGNARROW} @classmethod def set_argparser(cls, parser): narrows = parser.add_mutually_exclusive_group() narrows.add_argument('--all', dest='_packages_action', action='store_const', const='all', default=None, help=_("show all packages (default)")) narrows.add_argument('--available', dest='_packages_action', action='store_const', const='available', help=_("show only available packages")) narrows.add_argument('--installed', dest='_packages_action', action='store_const', const='installed', help=_("show only installed packages")) narrows.add_argument('--extras', dest='_packages_action', action='store_const', const='extras', help=_("show only extras packages")) narrows.add_argument('--updates', dest='_packages_action', action='store_const', const='upgrades', help=_("show only upgrades packages")) narrows.add_argument('--upgrades', dest='_packages_action', action='store_const', const='upgrades', help=_("show only upgrades packages")) narrows.add_argument('--autoremove', dest='_packages_action', action='store_const', const='autoremove', help=_("show only autoremove packages")) narrows.add_argument('--recent', dest='_packages_action', action='store_const', const='recent', help=_("show only recently changed packages")) parser.add_argument('packages', nargs='*', metavar=_('PACKAGE'), choices=cls.pkgnarrows, default=cls.DEFAULT_PKGNARROW, action=OptionParser.PkgNarrowCallback, help=_("Package name specification")) def configure(self): demands = self.cli.demands demands.sack_activation = True if self.opts._packages_action: self.opts.packages_action = self.opts._packages_action if self.opts.packages_action != 'installed': demands.available_repos = True if self.opts.obsoletes: if self.opts._packages_action: self.cli._option_conflict("--obsoletes", "--" + self.opts._packages_action) else: self.opts.packages_action = 'obsoletes' if self.opts.packages_action == 'updates': self.opts.packages_action = 'upgrades' def run(self): self.cli._populate_update_security_filter(self.opts) return self.base.output_packages('info', self.opts.packages_action, self.opts.packages) class ListCommand(InfoCommand): """A class containing methods needed by the cli to execute the list command. """ aliases = ('list', 'ls') summary = _('list a package or groups of packages') def run(self): self.cli._populate_update_security_filter(self.opts) return self.base.output_packages('list', self.opts.packages_action, self.opts.packages) class ProvidesCommand(Command): """A class containing methods needed by the cli to execute the provides command. """ aliases = ('provides', 'whatprovides', 'prov', 'wp') summary = _('find what package provides the given value') @staticmethod def set_argparser(parser): parser.add_argument('dependency', nargs='+', metavar=_('PROVIDE'), help=_("Provide specification to search for")) def configure(self): demands = self.cli.demands demands.available_repos = True demands.fresh_metadata = False demands.sack_activation = True def run(self): logger.debug(_("Searching Packages: ")) return self.base.provides(self.opts.dependency) class CheckUpdateCommand(Command): """A class containing methods needed by the cli to execute the check-update command. """ aliases = ('check-update', 'check-upgrade') summary = _('check for available package upgrades') @staticmethod def set_argparser(parser): parser.add_argument('--changelogs', dest='changelogs', default=False, action='store_true', help=_('show changelogs before update')) parser.add_argument('packages', nargs='*', metavar=_('PACKAGE')) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.plugin_filtering_enabled = True if self.opts.changelogs: demands.changelogs = True _checkEnabledRepo(self.base) def run(self): self.cli._populate_update_security_filter(self.opts, cmp_type="gte") found = self.base.check_updates(self.opts.packages, print_=True, changelogs=self.opts.changelogs) if found: self.cli.demands.success_exit_status = 100 if self.base.conf.autocheck_running_kernel: self.cli._check_running_kernel() class RepoPkgsCommand(Command): """Implementation of the repository-packages command.""" class CheckUpdateSubCommand(Command): """Implementation of the info sub-command.""" aliases = ('check-update',) def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" found = self.base.check_updates(self.opts.pkg_specs, self.reponame, print_=True) if found: self.cli.demands.success_exit_status = 100 class InfoSubCommand(Command): """Implementation of the info sub-command.""" aliases = ('info',) def configure(self): demands = self.cli.demands demands.sack_activation = True if self.opts._pkg_specs_action: self.opts.pkg_specs_action = self.opts._pkg_specs_action if self.opts.pkg_specs_action != 'installed': demands.available_repos = True if self.opts.obsoletes: if self.opts._pkg_specs_action: self.cli._option_conflict("--obsoletes", "--" + self.opts._pkg_specs_action) else: self.opts.pkg_specs_action = 'obsoletes' def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" self.cli._populate_update_security_filter(self.opts) self.base.output_packages('info', self.opts.pkg_specs_action, self.opts.pkg_specs, self.reponame) class InstallSubCommand(Command): """Implementation of the install sub-command.""" aliases = ('install',) def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_repo(self): self.cli._populate_update_security_filter(self.opts) """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Install all packages. try: self.base.install('*', self.reponame) except dnf.exceptions.MarkingError: logger.info(_('No package available.')) else: done = True else: # Install packages. for pkg_spec in self.opts.pkg_specs: try: self.base.install(pkg_spec, self.reponame) except dnf.exceptions.MarkingError as e: msg = '{}: {}'.format(e.value, self.base.output.term.bold(pkg_spec)) logger.info(msg) else: done = True if not done: raise dnf.exceptions.Error(_('No packages marked for install.')) class ListSubCommand(InfoSubCommand): """Implementation of the list sub-command.""" aliases = ('list',) def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" self.cli._populate_update_security_filter(self.opts) self.base.output_packages('list', self.opts.pkg_specs_action, self.opts.pkg_specs, self.reponame) class MoveToSubCommand(Command): """Implementation of the move-to sub-command.""" aliases = ('move-to',) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Reinstall all packages. try: self.base.reinstall('*', new_reponame=self.reponame) except dnf.exceptions.PackagesNotInstalledError: logger.info(_('No package installed.')) except dnf.exceptions.PackagesNotAvailableError: logger.info(_('No package available.')) except dnf.exceptions.MarkingError: assert False, 'Only the above marking errors are expected.' else: done = True else: # Reinstall packages. for pkg_spec in self.opts.pkg_specs: try: self.base.reinstall(pkg_spec, new_reponame=self.reponame) except dnf.exceptions.PackagesNotInstalledError: msg = _('No match for argument: %s') logger.info(msg, pkg_spec) except dnf.exceptions.PackagesNotAvailableError as err: for pkg in err.packages: xmsg = '' pkgrepo = self.base.history.repo(pkg) if pkgrepo: xmsg = _(' (from %s)') % pkgrepo msg = _('Installed package %s%s not available.') logger.info(msg, self.output.term.bold(pkg), xmsg) except dnf.exceptions.MarkingError: assert False, \ 'Only the above marking errors are expected.' else: done = True if not done: raise dnf.exceptions.Error(_('Nothing to do.')) class ReinstallOldSubCommand(Command): """Implementation of the reinstall-old sub-command.""" aliases = ('reinstall-old',) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Reinstall all packages. try: self.base.reinstall('*', self.reponame, self.reponame) except dnf.exceptions.PackagesNotInstalledError: msg = _('No package installed from the repository.') logger.info(msg) except dnf.exceptions.PackagesNotAvailableError: logger.info(_('No package available.')) except dnf.exceptions.MarkingError: assert False, 'Only the above marking errors are expected.' else: done = True else: # Reinstall packages. for pkg_spec in self.opts.pkg_specs: try: self.base.reinstall(pkg_spec, self.reponame, self.reponame) except dnf.exceptions.PackagesNotInstalledError: msg = _('No match for argument: %s') logger.info(msg, pkg_spec) except dnf.exceptions.PackagesNotAvailableError as err: for pkg in err.packages: xmsg = '' pkgrepo = self.base.history.repo(pkg) if pkgrepo: xmsg = _(' (from %s)') % pkgrepo msg = _('Installed package %s%s not available.') logger.info(msg, self.output.term.bold(pkg), xmsg) except dnf.exceptions.MarkingError: assert False, \ 'Only the above marking errors are expected.' else: done = True if not done: raise dnf.exceptions.Error(_('Nothing to do.')) class ReinstallSubCommand(Command): """Implementation of the reinstall sub-command.""" aliases = ('reinstall',) def __init__(self, cli): """Initialize the command.""" super(RepoPkgsCommand.ReinstallSubCommand, self).__init__(cli) self.wrapped_commands = (RepoPkgsCommand.ReinstallOldSubCommand(cli), RepoPkgsCommand.MoveToSubCommand(cli)) def configure(self): self.cli.demands.available_repos = True for command in self.wrapped_commands: command.opts = self.opts command.reponame = self.reponame command.configure() def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) for command in self.wrapped_commands: try: command.run_on_repo() except dnf.exceptions.Error: continue else: break else: raise dnf.exceptions.Error(_('No packages marked for reinstall.')) class RemoveOrDistroSyncSubCommand(Command): """Implementation of the remove-or-distro-sync sub-command.""" aliases = ('remove-or-distro-sync',) def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def _replace(self, pkg_spec, reponame): """Synchronize a package with another repository or remove it.""" self.cli.base.sack.disable_repo(reponame) subject = dnf.subject.Subject(pkg_spec) matches = subject.get_best_query(self.cli.base.sack) history = self.cli.base.history installed = [ pkg for pkg in matches.installed() if history.repo(pkg) == reponame] if not installed: raise dnf.exceptions.PackagesNotInstalledError( 'no package matched', pkg_spec) available = matches.available() clean_deps = self.cli.base.conf.clean_requirements_on_remove for package in installed: if available.filter(name=package.name, arch=package.arch): self.cli.base._goal.distupgrade(package) else: self.cli.base._goal.erase(package, clean_deps=clean_deps) def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Sync all packages. try: self._replace('*', self.reponame) except dnf.exceptions.PackagesNotInstalledError: msg = _('No package installed from the repository.') logger.info(msg) else: done = True else: # Reinstall packages. for pkg_spec in self.opts.pkg_specs: try: self._replace(pkg_spec, self.reponame) except dnf.exceptions.PackagesNotInstalledError: msg = _('No match for argument: %s') logger.info(msg, pkg_spec) else: done = True if not done: raise dnf.exceptions.Error(_('Nothing to do.')) class RemoveOrReinstallSubCommand(Command): """Implementation of the remove-or-reinstall sub-command.""" aliases = ('remove-or-reinstall',) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Reinstall all packages. try: self.base.reinstall('*', old_reponame=self.reponame, new_reponame_neq=self.reponame, remove_na=True) except dnf.exceptions.PackagesNotInstalledError: msg = _('No package installed from the repository.') logger.info(msg) except dnf.exceptions.MarkingError: assert False, 'Only the above marking error is expected.' else: done = True else: # Reinstall packages. for pkg_spec in self.opts.pkg_specs: try: self.base.reinstall( pkg_spec, old_reponame=self.reponame, new_reponame_neq=self.reponame, remove_na=True) except dnf.exceptions.PackagesNotInstalledError: msg = _('No match for argument: %s') logger.info(msg, pkg_spec) except dnf.exceptions.MarkingError: assert False, 'Only the above marking error is expected.' else: done = True if not done: raise dnf.exceptions.Error(_('Nothing to do.')) class RemoveSubCommand(Command): """Implementation of the remove sub-command.""" aliases = ('remove',) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.allow_erasing = True demands.available_repos = False demands.resolving = True demands.root_user = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" done = False if not self.opts.pkg_specs: # Remove all packages. try: self.base.remove('*', self.reponame) except dnf.exceptions.MarkingError: msg = _('No package installed from the repository.') logger.info(msg) else: done = True else: # Remove packages. for pkg_spec in self.opts.pkg_specs: try: self.base.remove(pkg_spec, self.reponame) except dnf.exceptions.MarkingError as e: logger.info(str(e)) else: done = True if not done: logger.warning(_('No packages marked for removal.')) class UpgradeSubCommand(Command): """Implementation of the upgrade sub-command.""" aliases = ('upgrade', 'upgrade-to') def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True def run_on_repo(self): """Execute the command with respect to given arguments *cli_args*.""" _checkGPGKey(self.base, self.cli) done = False if not self.opts.pkg_specs: # Update all packages. self.base.upgrade_all(self.reponame) done = True else: # Update packages. for pkg_spec in self.opts.pkg_specs: try: self.base.upgrade(pkg_spec, self.reponame) except dnf.exceptions.MarkingError: logger.info(_('No match for argument: %s'), pkg_spec) else: done = True if not done: raise dnf.exceptions.Error(_('No packages marked for upgrade.')) SUBCMDS = {CheckUpdateSubCommand, InfoSubCommand, InstallSubCommand, ListSubCommand, MoveToSubCommand, ReinstallOldSubCommand, ReinstallSubCommand, RemoveOrDistroSyncSubCommand, RemoveOrReinstallSubCommand, RemoveSubCommand, UpgradeSubCommand} aliases = ('repository-packages', 'repo-pkgs', 'repo-packages', 'repository-pkgs') summary = _('run commands on top of all packages in given repository') def __init__(self, cli): """Initialize the command.""" super(RepoPkgsCommand, self).__init__(cli) subcmd_objs = (subcmd(cli) for subcmd in self.SUBCMDS) self.subcmd = None self._subcmd_name2obj = { alias: subcmd for subcmd in subcmd_objs for alias in subcmd.aliases} def set_argparser(self, parser): narrows = parser.add_mutually_exclusive_group() narrows.add_argument('--all', dest='_pkg_specs_action', action='store_const', const='all', default=None, help=_("show all packages (default)")) narrows.add_argument('--available', dest='_pkg_specs_action', action='store_const', const='available', help=_("show only available packages")) narrows.add_argument('--installed', dest='_pkg_specs_action', action='store_const', const='installed', help=_("show only installed packages")) narrows.add_argument('--extras', dest='_pkg_specs_action', action='store_const', const='extras', help=_("show only extras packages")) narrows.add_argument('--updates', dest='_pkg_specs_action', action='store_const', const='upgrades', help=_("show only upgrades packages")) narrows.add_argument('--upgrades', dest='_pkg_specs_action', action='store_const', const='upgrades', help=_("show only upgrades packages")) narrows.add_argument('--autoremove', dest='_pkg_specs_action', action='store_const', const='autoremove', help=_("show only autoremove packages")) narrows.add_argument('--recent', dest='_pkg_specs_action', action='store_const', const='recent', help=_("show only recently changed packages")) parser.add_argument( 'reponame', nargs=1, action=OptionParser._RepoCallbackEnable, metavar=_('REPOID'), help=_("Repository ID")) subcommand_choices = [subcmd.aliases[0] for subcmd in self.SUBCMDS] subcommand_choices_all = [alias for subcmd in self.SUBCMDS for alias in subcmd.aliases] parser.add_argument('subcmd', nargs=1, metavar="SUBCOMMAND", choices=subcommand_choices_all, help=", ".join(subcommand_choices)) DEFAULT_PKGNARROW = 'all' pkgnarrows = {DEFAULT_PKGNARROW, 'installed', 'available', 'autoremove', 'extras', 'obsoletes', 'recent', 'upgrades'} parser.add_argument('pkg_specs', nargs='*', metavar=_('PACKAGE'), choices=pkgnarrows, default=DEFAULT_PKGNARROW, action=OptionParser.PkgNarrowCallback, help=_("Package specification")) def configure(self): """Verify whether the command can run with given arguments.""" # Check sub-command. try: self.subcmd = self._subcmd_name2obj[self.opts.subcmd[0]] except (dnf.cli.CliError, KeyError) as e: self.cli.optparser.print_usage() raise dnf.cli.CliError self.subcmd.opts = self.opts self.subcmd.reponame = self.opts.reponame[0] self.subcmd.configure() def run(self): """Execute the command with respect to given arguments *extcmds*.""" self.subcmd.run_on_repo() class HelpCommand(Command): """A class containing methods needed by the cli to execute the help command. """ aliases = ('help',) summary = _('display a helpful usage message') @staticmethod def set_argparser(parser): parser.add_argument('cmd', nargs='?', metavar=_('COMMAND'), help=_("{prog} command to get help for").format( prog=dnf.util.MAIN_PROG_UPPER)) def run(self): if (not self.opts.cmd or self.opts.cmd not in self.cli.cli_commands): self.cli.optparser.print_help() else: command = self.cli.cli_commands[self.opts.cmd] self.cli.optparser.print_help(command(self)) cp.py 0000644 00000162765 15025166704 0005547 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cp command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import itertools import logging import os import time import traceback from apitools.base.py import encoding from gslib import gcs_json_api from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import DestinationInfo from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import NameExpansionIteratorDestinationTuple from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import ContainsWildcard from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import cat_helper from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS from gslib.utils.constants import NO_MAX from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import ItemExistsError from gslib.utils.copy_helper import Manifest from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import GetStreamFromFileUrl from gslib.utils.system_util import StdinIterator from gslib.utils.system_util import StdinIteratorCls from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.text_util import RemoveCRLFFromString from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import MakeHumanReadable _SYNOPSIS = """ gsutil cp [OPTION]... src_url dst_url gsutil cp [OPTION]... src_url... dst_url gsutil cp [OPTION]... -I dst_url """ _SYNOPSIS_TEXT = """ <B>SYNOPSIS</B> """ + _SYNOPSIS _DESCRIPTION_TEXT = """ <B>DESCRIPTION</B> The ``gsutil cp`` command allows you to copy data between your local file system and the cloud, within the cloud, and between cloud storage providers. For example, to upload all text files from the local directory to a bucket, you can run: gsutil cp *.txt gs://my-bucket You can also download data from a bucket. The following command downloads all text files from the top-level of a bucket to your current directory: gsutil cp gs://my-bucket/*.txt . You can use the ``-n`` option to prevent overwriting the content of existing files. The following example downloads text files from a bucket without clobbering the data in your directory: gsutil cp -n gs://my-bucket/*.txt . Use the ``-r`` option to copy an entire directory tree. For example, to upload the directory tree ``dir``: gsutil cp -r dir gs://my-bucket If you have a large number of files to transfer, you can perform a parallel multi-threaded/multi-processing copy using the top-level gsutil ``-m`` option (see "gsutil help options"): gsutil -m cp -r dir gs://my-bucket You can use the ``-I`` option with ``stdin`` to specify a list of URLs to copy, one per line. This allows you to use gsutil in a pipeline to upload or download objects as generated by a program: cat filelist | gsutil -m cp -I gs://my-bucket or: cat filelist | gsutil -m cp -I ./download_dir where the output of ``cat filelist`` is a list of files, cloud URLs, and wildcards of files and cloud URLs. NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand wildcards in ways that can be surprising. You may also encounter issues when attempting to copy files whose names contain wildcard characters. For more details about these issues, see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. """ _NAME_CONSTRUCTION_TEXT = """ <B>HOW NAMES ARE CONSTRUCTED</B> The ``gsutil cp`` command attempts to name objects in ways that are consistent with the Linux ``cp`` command. This means that names are constructed depending on whether you're performing a recursive directory copy or copying individually-named objects, or whether you're copying to an existing or non-existent directory. When you perform recursive directory copies, object names are constructed to mirror the source directory structure starting at the point of recursive processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the following command creates the object ``gs://my-bucket/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket In contrast, copying individually-named files results in objects named by the final path component of the source files. For example, assuming again that ``dir1/dir2`` contains ``a/b/c``, the following command creates the object ``gs://my-bucket/c``: gsutil cp dir1/dir2/** gs://my-bucket Note that in the above example, the '**' wildcard matches all names anywhere under ``dir``. The wildcard '*' matches names just one level deep. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. The same rules apply for uploads and downloads: recursive copies of buckets and bucket subdirectories produce a mirrored filename structure, while copying individually or wildcard-named objects produce flatly-named files. In addition, the resulting names depend on whether the destination subdirectory exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory, the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket/subdir In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp`` command creates the object ``gs://my-bucket/subdir/a/b/c``. NOTE: The `Google Cloud Platform Console <https://console.cloud.google.com>`_ creates folders by creating "placeholder" objects that end with a "/" character. gsutil skips these objects when downloading from the cloud to the local file system, because creating a file that ends with a "/" is not allowed on Linux and macOS. We recommend that you only create objects that end with "/" if you don't intend to download such objects using gsutil. """ _SUBDIRECTORIES_TEXT = """ <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> You can use gsutil to copy to and from subdirectories by using a command like this: gsutil cp -r dir gs://my-bucket/data This causes ``dir`` and all of its files and nested subdirectories to be copied under the specified destination, resulting in objects with names like ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket subdirectories using the following command: gsutil cp -r gs://my-bucket/data dir This causes everything nested under ``gs://my-bucket/data`` to be downloaded into ``dir``, resulting in files with names like ``dir/data/a/b/c``. Copying subdirectories is useful if you want to add data to an existing bucket directory structure over time. It's also useful if you want to parallelize uploads and downloads across multiple machines (potentially reducing overall transfer time compared with running ``gsutil -m cp`` on one machine). For example, if your bucket contains this structure: gs://my-bucket/data/result_set_01/ gs://my-bucket/data/result_set_02/ ... gs://my-bucket/data/result_set_99/ you can perform concurrent downloads across 3 machines by running these commands on each machine, respectively: gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir Note that ``dir`` could be a local directory on each machine, or a directory mounted off of a shared file server. The performance of the latter depends on several factors, so we recommend experimenting to find out what works best for your computing environment. """ _COPY_IN_CLOUD_TEXT = """ <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast, when you download data from the cloud, it ends up in a file with no associated metadata, unless you have some way to keep or re-create that metadata. Copies spanning locations and/or storage classes cause data to be rewritten in the cloud, which may take some time (but is still faster than downloading and re-uploading). Such operations can be resumed with the same command if they are interrupted, so long as the command parameters are identical. Note that by default, the gsutil ``cp`` command does not copy the object ACL to the new object, and instead uses the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the ``-p`` option. When copying in the cloud, if the destination bucket has Object Versioning enabled, by default ``gsutil cp`` copies only live versions of the source object. For example, the following command causes only the single live version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there are noncurrent versions of ``gs://bucket1/obj``: gsutil cp gs://bucket1/obj gs://bucket2 To also copy noncurrent versions, use the ``-A`` flag: gsutil cp -A gs://bucket1/obj gs://bucket2 The top-level gsutil ``-m`` flag is not allowed when using the ``cp -A`` flag. """ _CHECKSUM_VALIDATION_TEXT = """ <B>CHECKSUM VALIDATION</B> gsutil automatically performs checksum validation for copies to and from Cloud Storage. For more information, see `Hashes and ETags <https://cloud.google.com/storage/docs/hashes-etags#cli>`_. """ _RETRY_HANDLING_TEXT = """ <B>RETRY HANDLING</B> The ``cp`` command retries when failures occur, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the ``cp`` command skips that object and moves on. If any failures were not successfully retried by the end of the copy run, the ``cp`` command reports the number of failures and exits with a non-zero status. For details about gsutil's overall retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. """ _RESUMABLE_TRANSFERS_TEXT = """ <B>RESUMABLE TRANSFERS</B> gsutil automatically resumes interrupted downloads and interrupted `resumable uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_, except when performing streaming transfers. In the case of an interrupted download, a partially downloaded temporary file is visible in the destination directory with the suffix ``_.gstmp`` in its name. Upon completion, the original file is deleted and replaced with the downloaded contents. Resumable transfers store state information in files under ~/.gsutil, named by the destination object or file. See "gsutil help prod" for details on using resumable transfers in production. """ _STREAMING_TRANSFERS_TEXT = """ <B>STREAMING TRANSFERS</B> Use '-' in place of src_url or dst_url to perform a `streaming transfer <https://cloud.google.com/storage/docs/streaming>`_. Streaming uploads using the `JSON API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered in memory part-way back into the file and can thus sometimes resume in the event of network or service problems. gsutil does not support resuming streaming uploads using the XML API or resuming streaming downloads for either JSON or XML. If you have a large amount of data to transfer in these cases, we recommend that you write the data to a local file and copy that file rather than streaming it. """ _SLICED_OBJECT_DOWNLOADS_TEXT = """ <B>SLICED OBJECT DOWNLOADS</B> gsutil can automatically use ranged ``GET`` requests to perform downloads in parallel for large files being downloaded from Cloud Storage. See `sliced object download documentation <https://cloud.google.com/storage/docs/sliced-object-downloads>`_ for a complete discussion. """ _PARALLEL_COMPOSITE_UPLOADS_TEXT = """ <B>PARALLEL COMPOSITE UPLOADS</B> gsutil can automatically use `object composition <https://cloud.google.com/storage/docs/composite-objects>`_ to perform uploads in parallel for large, local files being uploaded to Cloud Storage. See the `parallel composite uploads documentation <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a complete discussion. """ _CHANGING_TEMP_DIRECTORIES_TEXT = """ <B>CHANGING TEMP DIRECTORIES</B> gsutil writes data to a temporary directory in several cases: - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options) - when decompressing data being downloaded (for example, when the data has ``Content-Encoding:gzip`` as a result of being uploaded using gsutil cp -z or gsutil cp -Z) - when running integration tests using the gsutil test command In these cases, it's possible the temporary file location on your system that gsutil selects by default may not have enough space. If gsutil runs out of space during one of these operations (for example, raising "CommandException: Inadequate temp space available to compress <your file>" during a ``gsutil cp -z`` operation), you can change where it writes these temp files by setting the TMPDIR environment variable. On Linux and macOS, you can set the variable as follows: TMPDIR=/some/directory gsutil cp ... You can also add this line to your ~/.bashrc file and restart the shell before running gsutil: export TMPDIR=/some/directory On Windows 7, you can change the TMPDIR environment variable from Start -> Computer -> System -> Advanced System Settings -> Environment Variables. You need to reboot after making this change for it to take effect. Rebooting is not necessary after running the export command on Linux and macOS. """ _COPYING_SPECIAL_FILES_TEXT = """ <B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B> Please see the section about OS-specific file types in "gsutil help rsync". While that section refers to the ``rsync`` command, analogous points apply to the ``cp`` command. """ _OPTIONS_TEXT = """ <B>OPTIONS</B> -a predef_acl Applies the specific predefined ACL to uploaded objects. See "gsutil help acls" for further details. -A Copy all source versions from a source bucket or folder. If not set, only the live version of each source object is copied. NOTE: This option is only useful when the destination bucket has Object Versioning enabled. Additionally, the generation numbers of copied versions do not necessarily match the order of the original generation numbers. -c If an error occurs, continue attempting to copy the remaining files. If any copies are unsuccessful, gsutil's exit status is non-zero, even if this flag is set. This option is implicitly set when running ``gsutil -m cp...``. NOTE: ``-c`` only applies to the actual copying operation. If an error, such as ``invalid Unicode file name``, occurs while iterating over the files in the local directory, gsutil prints an error message and aborts. -D Copy in "daisy chain" mode, which means copying between two buckets by first downloading to the machine where gsutil is run, then uploading to the destination bucket. The default mode is a "copy in the cloud," where data is copied between two buckets without uploading or downloading. During a "copy in the cloud," a source composite object remains composite at its destination. However, you can use "daisy chain" mode to change a composite object into a non-composite object. For example: gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp gsutil mv gs://bucket/obj_tmp gs://bucket/obj NOTE: "Daisy chain" mode is automatically used when copying between providers: for example, when copying data from Cloud Storage to another provider. -e Exclude symlinks. When specified, symbolic links are not copied. -I Use ``stdin`` to specify a list of files or objects to copy. You can use gsutil in a pipeline to upload or download objects as generated by a program. For example: cat filelist | gsutil -m cp -I gs://my-bucket where the output of ``cat filelist`` is a one-per-line list of files, cloud URLs, and wildcards of files and cloud URLs. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the ``-j`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files. This also saves network bandwidth while leaving the data uncompressed in Cloud Storage. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the ``Content-Type`` and name of the original files. Note that if you want to use the ``-m`` `top-level option <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_ to parallelize copies along with the ``-j/-J`` options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. You can change this compression buffer size to a higher limit. For example: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m cp -j html,txt -r /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the ``-j`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in longer uploads. -L <file> Outputs a manifest log file with detailed information about each item that was copied. This manifest contains the following information for each item: - Source path. - Destination path. - Source size. - Bytes transferred. - MD5 hash. - Transfer start time and date in UTC and ISO 8601 format. - Transfer completion time and date in UTC and ISO 8601 format. - Upload id, if a resumable upload was performed. - Final result of the attempted transfer, either success or failure. - Failure details, if any. If the log file already exists, gsutil uses the file as an input to the copy process, and appends log items to the existing file. Objects that are marked in the existing log file as having been successfully copied or skipped are ignored. Objects without entries are copied and ones previously marked as unsuccessful are retried. This option can be used in conjunction with the ``-c`` option to build a script that copies a large number of objects reliably, using a bash script like the following: until gsutil cp -c -L cp.log -r ./dir gs://bucket; do sleep 1 done The -c option enables copying to continue after failures occur, and the -L option allows gsutil to pick up where it left off without duplicating work. The loop continues running as long as gsutil exits with a non-zero status. A non-zero status indicates there was at least one failure during the copy operation. NOTE: If you are synchronizing the contents of a directory and a bucket, or the contents of two buckets, see "gsutil help rsync". -n No-clobber. When specified, existing files or objects at the destination are not replaced. Any items that are skipped by this option are reported as skipped. gsutil performs an additional GET request to check if an item exists before attempting to upload the data. This saves gsutil from retransmitting data, but the additional HTTP requests may make small object transfers slower and more expensive. -p Preserves ACLs when copying in the cloud. Note that this option has performance and cost implications only when using the XML API, as the XML API requires separate HTTP calls for interacting with ACLs. You can mitigate this performance issue using ``gsutil -m cp`` to perform parallel copying. Note that this option only works if you have OWNER access to all objects that are copied. If you want all objects in the destination bucket to end up with the same ACL, you can avoid these performance issues by setting a default object ACL on that bucket instead of using ``cp -p``. See "gsutil help defacl". Note that it's not valid to specify both the ``-a`` and ``-p`` options together. -P Enables POSIX attributes to be preserved when objects are copied. ``gsutil cp`` copies fields provided by ``stat``. These fields are the user ID of the owner, the group ID of the owning group, the mode or permissions of the file, and the access and modification time of the file. For downloads, these attributes are only set if the source objects were uploaded with this flag enabled. On Windows, this flag only sets and restores access time and modification time. This is because Windows doesn't support POSIX uid/gid/mode. -R, -r The ``-R`` and ``-r`` options are synonymous. They enable directories, buckets, and bucket subdirectories to be copied recursively. If you don't use this option for an upload, gsutil copies objects it finds and skips directories. Similarly, if you don't specify this option for a download, gsutil copies objects at the current bucket directory level and skips subdirectories. -s <class> Specifies the storage class of the destination object. If not specified, the default storage class of the destination bucket is used. This option is not valid for copying to non-cloud destinations. -U Skips objects with unsupported object types instead of failing. Unsupported object types include Amazon S3 objects in the GLACIER storage class. -v Prints the version-specific URL for each uploaded object. You can use these URLs to safely make concurrent upload requests, because Cloud Storage refuses to perform an update if the current object version doesn't match the version-specific URL. See `generation numbers <https://cloud.google.com/storage/docs/metadata#generation-number>`_ for more details. -z <ext,...> Applies gzip content-encoding to any file upload whose extension matches the ``-z`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files, because it reduces network bandwidth and storage sizes. This can both improve performance and reduce costs. When you specify the ``-z`` option, the data from your files is compressed before it is uploaded, but your actual files are left uncompressed on the local disk. The uploaded objects retain the ``Content-Type`` and name of the original files, but have their ``Content-Encoding`` metadata set to ``gzip`` to indicate that the object data stored are compressed on the Cloud Storage servers and have their ``Cache-Control`` metadata set to ``no-transform``. For example, the following command: gsutil cp -z html \\ cattypes.html tabby.jpeg gs://mycats does the following: - The ``cp`` command uploads the files ``cattypes.html`` and ``tabby.jpeg`` to the bucket ``gs://mycats``. - Based on the file extensions, gsutil sets the ``Content-Type`` of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to ``image/jpeg``. - The ``-z`` option compresses the data in the file ``cattypes.html``. - The ``-z`` option also sets the ``Content-Encoding`` for ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for ``cattypes.html`` to ``no-transform``. Because the ``-z/-Z`` options compress data prior to upload, they are not subject to the same compression buffer bottleneck that can affect the ``-j/-J`` options. Note that if you download an object with ``Content-Encoding:gzip``, gsutil decompresses the content before writing the local file. -Z Applies gzip content-encoding to file uploads. This option works like the ``-z`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in files taking up more space in the cloud than they would if left uncompressed. --stet If the STET binary can be found in boto or PATH, cp will use the split-trust encryption tool for end-to-end encryption. """ _DETAILED_HELP_TEXT = '\n\n'.join([ _SYNOPSIS_TEXT, _DESCRIPTION_TEXT, _NAME_CONSTRUCTION_TEXT, _SUBDIRECTORIES_TEXT, _COPY_IN_CLOUD_TEXT, _CHECKSUM_VALIDATION_TEXT, _RETRY_HANDLING_TEXT, _RESUMABLE_TRANSFERS_TEXT, _STREAMING_TRANSFERS_TEXT, _SLICED_OBJECT_DOWNLOADS_TEXT, _PARALLEL_COMPOSITE_UPLOADS_TEXT, _CHANGING_TEMP_DIRECTORIES_TEXT, _COPYING_SPECIAL_FILES_TEXT, _OPTIONS_TEXT, ]) CP_SUB_ARGS = 'a:AcDeIL:MNnpPrRs:tUvz:Zj:J' # May be used by cp or mv. CP_AND_MV_SHIM_FLAG_MAP = { '-A': GcloudStorageFlag('--all-versions'), '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--continue-on-error'), '-D': GcloudStorageFlag('--daisy-chain'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-I': GcloudStorageFlag('--read-paths-from-stdin'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-L': GcloudStorageFlag('--manifest-path'), '-n': GcloudStorageFlag('--no-clobber'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-s': GcloudStorageFlag('--storage-class'), '-v': GcloudStorageFlag('--print-created-message'), '-Z': GcloudStorageFlag('--gzip-local-all'), '-z': GcloudStorageFlag('--gzip-local'), } # Adds recursion flags. CP_SHIM_FLAG_MAP = { k: v for k, v in list(CP_AND_MV_SHIM_FLAG_MAP.items()) + [('-r', GcloudStorageFlag('-r')), ('-R', GcloudStorageFlag('-r'))] } def ShimTranslatePredefinedAclSubOptForCopy(sub_opts): """Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case. The camel-case-snake-case difference is related to gcloud primarily using JSON API rather than the XML API. Predefined ACLs are also called "canned ACLs". Args: sub_opts: List of pairs representing flag keys and values, e.g. [('a', 'public-read')] """ predefined_acl_idx = None for i, (k, _) in enumerate(sub_opts): if k == '-a': predefined_acl_idx = i break if predefined_acl_idx is not None: old_predefined_acl = sub_opts[i][1] sub_opts[i] = (sub_opts[i][0], gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.get( old_predefined_acl, old_predefined_acl)) def _CopyFuncWrapper(cls, args, thread_state=None): cls.CopyFunc(args, thread_state=thread_state, preserve_posix=cls.preserve_posix_attrs) def _CopyExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while copying:\n%s\n', traceback.format_exc()) def _RmExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) class CpCommand(Command): """Implementation of gsutil cp command. Note that CpCommand is run for both gsutil cp and gsutil mv. The latter happens by MvCommand calling CpCommand and passing the hidden (undocumented) -M option. This allows the copy and remove needed for each mv to run together (rather than first running all the cp's and then all the rm's, as we originally had implemented), which in turn avoids the following problem with removing the wrong objects: starting with a bucket containing only the object gs://bucket/obj, say the user does: gsutil mv gs://bucket/* gs://bucket/d.txt If we ran all the cp's and then all the rm's and we didn't expand the wildcard first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, and the rm command would then remove that object. In the implementation prior to gsutil release 3.12 we avoided this by building a list of objects to process and then running the copies and then the removes; but building the list up front limits scalability (compared with the current approach of processing the bucket listing iterator on the fly). """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cp', command_name_aliases=['copy'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # -t is deprecated but leave intact for now to avoid breakage. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, # Unfortunately, "private" args are the only way to support non-single # character flags. supported_private_args=['stet', 'testcallbackfile='], argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cp', help_name_aliases=['copy'], help_type='command_help', help_one_line_summary='Copy files and objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): self.logger.warn( "Unlike pure gsutil, this shim won't run composite uploads and sliced" ' downloads in parallel by default. Use the -m flag to enable' ' parallelism (i.e. "gsutil -m cp ...").') ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cp'], flag_map=CP_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) # pylint: disable=too-many-statements def CopyFunc(self, copy_object_info, thread_state=None, preserve_posix=False): """Worker function for performing the actual copy (and rm, for mv).""" gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) copy_helper_opts = copy_helper.GetCopyHelperOpts() if copy_helper_opts.perform_mv: cmd_name = 'mv' else: cmd_name = self.command_name src_url = copy_object_info.source_storage_url exp_src_url = copy_object_info.expanded_storage_url src_url_names_container = copy_object_info.names_container have_multiple_srcs = copy_object_info.is_multi_source_request if src_url.IsCloudUrl() and src_url.IsProvider(): raise CommandException( 'The %s command does not allow provider-only source URLs (%s)' % (cmd_name, src_url)) if preserve_posix and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException('Cannot preserve POSIX attributes with a stream.') if self.parallel_operations and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException( 'Cannot upload from a stream when using gsutil -m option.') if have_multiple_srcs: copy_helper.InsistDstUrlNamesContainer( copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, cmd_name) # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. We # need to filter them out here, otherwise if the user tries to rsync from # GCS to a local directory it will result in a directory/file conflict # (e.g., trying to download an object called "mydata/" where the local # directory "mydata" exists). if IsCloudSubdirPlaceholder(exp_src_url): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. return if copy_helper_opts.use_manifest and self.manifest.WasSuccessful( exp_src_url.url_string): return if copy_helper_opts.perform_mv and copy_object_info.names_container: # Use recursion_requested when performing name expansion for the # directory mv case so we can determine if any of the source URLs are # directories (and then use cp -r and rm -r to perform the move, to # match the behavior of Linux mv (which when moving a directory moves # all the contained files). self.recursion_requested = True if (copy_object_info.exp_dst_url.IsFileUrl() and not os.path.exists(copy_object_info.exp_dst_url.object_name) and have_multiple_srcs): try: os.makedirs(copy_object_info.exp_dst_url.object_name) except OSError as e: if e.errno != errno.EEXIST: raise dst_url = copy_helper.ConstructDstUrl( src_url, exp_src_url, src_url_names_container, have_multiple_srcs, copy_object_info.is_multi_top_level_source_request, copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, self.recursion_requested, preserve_posix=preserve_posix) dst_url = copy_helper.FixWindowsNaming(src_url, dst_url) copy_helper.CheckForDirFileConflict(exp_src_url, dst_url) if copy_helper.SrcDstSame(exp_src_url, dst_url): raise CommandException('%s: "%s" and "%s" are the same file - ' 'abort.' % (cmd_name, exp_src_url, dst_url)) if dst_url.IsCloudUrl() and dst_url.HasGeneration(): raise CommandException('%s: a version-specific URL\n(%s)\ncannot be ' 'the destination for gsutil cp - abort.' % (cmd_name, dst_url)) if not dst_url.IsCloudUrl() and copy_helper_opts.dest_storage_class: raise CommandException('Cannot specify storage class for a non-cloud ' 'destination: %s' % dst_url) src_obj_metadata = None if copy_object_info.expanded_result: src_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, copy_object_info.expanded_result) if src_url.IsFileUrl() and preserve_posix: if not src_obj_metadata: src_obj_metadata = apitools_messages.Object() mode, _, _, _, uid, gid, _, atime, mtime, _ = os.stat( exp_src_url.object_name) mode = ConvertModeToBase8(mode) posix_attrs = POSIXAttributes(atime=atime, mtime=mtime, uid=uid, gid=gid, mode=mode) custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, custom_metadata, preserve_posix=preserve_posix) src_obj_metadata.metadata = custom_metadata if src_obj_metadata and dst_url.IsFileUrl(): posix_attrs = DeserializeFileAttributesFromObjectMetadata( src_obj_metadata, src_url.url_string) mode = posix_attrs.mode.permissions valid, err = ValidateFilePermissionAccess(src_url.url_string, uid=posix_attrs.uid, gid=posix_attrs.gid, mode=mode) if preserve_posix and not valid: logging.getLogger().critical(err) raise CommandException('This sync will orphan file(s), please fix their' ' permissions before trying again.') bytes_transferred = 0 try: if copy_helper_opts.use_manifest: self.manifest.Initialize(exp_src_url.url_string, dst_url.url_string) if (self.recursion_requested and copy_object_info.exp_dst_url.object_name and dst_url.IsFileUrl()): # exp_dst_url is the wildcard-expanded path passed by the user: # exp_dst_url => ~/dir # container => /usr/name/dir container = os.path.abspath(copy_object_info.exp_dst_url.object_name) # dst_url holds the complete path of the object's destination: # dst_url => /usr/name/dir/../file.txt # abspath => /usr/name/file.txt # # Taking the common path of this and container yields: /usr/name, # which does not start with container when the inclusion of '..' strings # results in a copy outside of the container. if not os.path.commonpath([ container, os.path.abspath(dst_url.object_name) ]).startswith(container): self.logger.warn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (exp_src_url, container)) if copy_helper_opts.use_manifest: self.manifest.SetResult( exp_src_url.url_string, 0, 'skip', 'Would have copied outside the destination directory.') return _, bytes_transferred, result_url, md5 = copy_helper.PerformCopy( self.logger, exp_src_url, dst_url, gsutil_api, self, _CopyExceptionHandler, src_obj_metadata=src_obj_metadata, allow_splitting=True, headers=self.headers, manifest=self.manifest, gzip_encoded=self.gzip_encoded, gzip_exts=self.gzip_exts, preserve_posix=preserve_posix, use_stet=self.use_stet) if copy_helper_opts.use_manifest: if md5: self.manifest.Set(exp_src_url.url_string, 'md5', md5) self.manifest.SetResult(exp_src_url.url_string, bytes_transferred, 'OK') if copy_helper_opts.print_ver: # Some cases don't return a version-specific URL (e.g., if destination # is a file). self.logger.info('Created: %s', result_url) except ItemExistsError: message = 'Skipping existing item: %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except SkipUnsupportedObjectError as e: message = ('Skipping item %s with unsupported object type %s' % (exp_src_url.url_string, e.unsupported_type)) self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except copy_helper.FileConcurrencySkipError as e: self.logger.warn( 'Skipping copy of source URL %s because destination URL ' '%s is already being copied by another gsutil process ' 'or thread (did you specify the same source URL twice?) ' % (src_url, dst_url)) except Exception as e: # pylint: disable=broad-except if (copy_helper_opts.no_clobber and copy_helper.IsNoClobberServerException(e)): message = 'Rejected (noclobber): %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) elif self.continue_on_error: message = 'Error copying %s: %s' % (src_url, str(e)) self.op_failure_count += 1 self.logger.error(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', RemoveCRLFFromString(message)) else: if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', str(e)) raise else: if copy_helper_opts.perform_mv: self.logger.info('Removing %s...', exp_src_url) if exp_src_url.IsCloudUrl(): gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, generation=exp_src_url.generation, provider=exp_src_url.scheme) else: os.unlink(exp_src_url.object_name) with self.stats_lock: # TODO: Remove stats_lock; we should be able to calculate bytes # transferred from StatusMessages posted by operations within PerformCopy. self.total_bytes_transferred += bytes_transferred def _ConstructNameExpansionIteratorDstTupleIterator(self, src_url_strs_iter, dst_url_strs): copy_helper_opts = copy_helper.GetCopyHelperOpts() for src_url_str, dst_url_str in zip(src_url_strs_iter, dst_url_strs): # Getting the destination information for each (sources, destination) # tuple. This assumes that the same destination is never provided in # multiple tuples, and doing so may result in an inconsistent behavior # especially when using the -m multi-threading option. # # Example for the inconsistent behavior, the following commands will # behave differently: # # gsutil cp -r dir1 dir2 gs://bucket/non-existent-dir # gsutil cp -r [ # (dir1, gs://bucket/non-existent-dir), # (dir2, gs://bucket/non-existent-dir) # ] # # When multiple threads execute on a non existing destination directory. # These threads might encounter different states of the destination # directory. The first thread to execute the command finds that the # destination directory does not exist, it will create the destination # directory and copies the files inside the source directories to the # destination directory. The following threads find that the destination # directory already exists and copy the source directories in the # destination directory. In another scenario, all the threads might find # that the destination directory does not exist and copy the source # directories to the destination directory. exp_dst_url, have_existing_dst_container = ( copy_helper.ExpandUrlToSingleBlr(dst_url_str, self.gsutil_api, self.project_id, logger=self.logger)) name_expansion_iterator_dst_tuple = NameExpansionIteratorDestinationTuple( NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, src_url_str, self.recursion_requested or copy_helper_opts.perform_mv, project_id=self.project_id, all_versions=self.all_versions, ignore_symlinks=self.exclude_symlinks, continue_on_error=(self.continue_on_error or self.parallel_operations), bucket_listing_fields=GetSourceFieldsNeededForCopy( exp_dst_url.IsCloudUrl(), copy_helper_opts.skip_unsupported_objects, copy_helper_opts.preserve_acl, preserve_posix=self.preserve_posix_attrs, delete_source=copy_helper_opts.perform_mv, file_size_will_change=self.use_stet)), DestinationInfo(exp_dst_url, have_existing_dst_container)) self.has_file_dst = self.has_file_dst or exp_dst_url.IsFileUrl() self.has_cloud_dst = self.has_cloud_dst or exp_dst_url.IsCloudUrl() self.provider_types.add(exp_dst_url.scheme) self.combined_src_urls = itertools.chain(self.combined_src_urls, src_url_str) yield name_expansion_iterator_dst_tuple # Command entry point. def RunCommand(self): copy_helper_opts = self._ParseOpts() self.total_bytes_transferred = 0 dst_url = StorageUrlFromString(self.args[-1]) if dst_url.IsFileUrl() and (dst_url.object_name == '-' or dst_url.IsFifo()): if self.preserve_posix_attrs: raise CommandException('Cannot preserve POSIX attributes with a ' 'stream or a named pipe.') cat_out_fd = (GetStreamFromFileUrl(dst_url, mode='wb') if dst_url.IsFifo() else None) return cat_helper.CatHelper(self).CatUrlStrings(self.args[:-1], cat_out_fd=cat_out_fd) if copy_helper_opts.read_args_from_stdin: if len(self.args) != 1: raise CommandException('Source URLs cannot be specified with -I option') # Use StdinIteratorCls instead of StdinIterator here to avoid Python 3 # generator pickling errors when multiprocessing a command. src_url_strs = [StdinIteratorCls()] else: if len(self.args) < 2: raise CommandException('Wrong number of arguments for "cp" command.') src_url_strs = [self.args[:-1]] dst_url_strs = [self.args[-1]] self.combined_src_urls = [] self.has_file_dst = False self.has_cloud_dst = False self.provider_types = set() # Because cp may have multiple source URLs and multiple destinations, we # wrap the name expansion iterator in order to collect analytics. name_expansion_iterator = CopyObjectsIterator( self._ConstructNameExpansionIteratorDstTupleIterator( src_url_strs, dst_url_strs), copy_helper_opts.daisy_chain, ) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=None, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, process_count * thread_count) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not copy_helper_opts.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.combined_src_urls, self.recursion_requested or copy_helper_opts.perform_mv, all_versions=self.all_versions, project_id=self.project_id, ignore_symlinks=self.exclude_symlinks, file_size_will_change=self.use_stet) # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() # Tracks if any copies failed. self.op_failure_count = 0 # Start the clock. start_time = time.time() # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') # Perform copy requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_CopyFuncWrapper, name_expansion_iterator, _CopyExceptionHandler, shared_attrs, fail_on_error=(not self.continue_on_error), seek_ahead_iterator=seek_ahead_iterator) self.logger.debug('total_bytes_transferred: %d', self.total_bytes_transferred) end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( has_file_dst=self.has_file_dst, has_cloud_dst=self.has_cloud_dst, avg_throughput=self.total_bytes_per_second, total_bytes_transferred=self.total_bytes_transferred, total_elapsed_time=self.total_elapsed_time, uses_fan=self.parallel_operations, is_daisy_chain=copy_helper_opts.daisy_chain, provider_types=list(self.provider_types)) if self.debug >= DEBUGLEVEL_DUMP_REQUESTS: # Note that this only counts the actual GET and PUT bytes for the copy # - not any transfers for doing wildcard expansion, the initial # HEAD/GET request performed to get the object metadata, etc. if self.total_bytes_transferred != 0: self.logger.info( 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)', self.total_bytes_transferred, self.total_elapsed_time, MakeHumanReadable(self.total_bytes_per_second)) if self.op_failure_count: plural_str = 's' if self.op_failure_count > 1 else '' raise CommandException('{count} file{pl}/object{pl} could ' 'not be transferred.'.format( count=self.op_failure_count, pl=plural_str)) return 0 def _ParseOpts(self): # TODO: Arrange variables initialized here in alphabetical order. perform_mv = False # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False no_clobber = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False daisy_chain = False read_args_from_stdin = False print_ver = False use_manifest = False preserve_acl = False self.preserve_posix_attrs = False canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None self.all_versions = False self.skip_unsupported_objects = False # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. The gzip_local flag marks if the files should be compressed # before uploading. Files compressed prior to uploaded are stored # compressed, while files compressed during the upload are stored # uncompressed. These flags cannot be mixed. gzip_encoded = False gzip_local = False gzip_arg_exts = None gzip_arg_all = None test_callback_file = None dest_storage_class = None self.use_stet = False # self.recursion_requested initialized in command.py (so can be checked # in parent class for all commands). self.manifest = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-A': self.all_versions = True if o == '-c': self.continue_on_error = True elif o == '-D': daisy_chain = True elif o == '-e': self.exclude_symlinks = True elif o == '--testcallbackfile': # File path of a pickled class that implements ProgressCallback.call. # Used for testing transfer interruptions and resumes. test_callback_file = a elif o == '-I': read_args_from_stdin = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-L': use_manifest = True self.manifest = Manifest(a) elif o == '-M': # Note that we signal to the cp command to perform a move (copy # followed by remove) and use directory-move naming rules by passing # the undocumented (for internal use) -M option when running the cp # command from mv.py. perform_mv = True elif o == '-n': no_clobber = True elif o == '-p': preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-s': dest_storage_class = NormalizeStorageClass(a) elif o == '-U': self.skip_unsupported_objects = True elif o == '-v': print_ver = True elif o == '-z': gzip_local = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-Z': gzip_local = True gzip_arg_all = GZIP_ALL_FILES elif o == '--stet': self.use_stet = True if preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if self.all_versions and self.parallel_operations: raise CommandException( 'The gsutil -m option is not supported with the cp -A flag, to ' 'ensure that object version ordering is preserved. Please re-run ' 'the command without the -m option.') if gzip_encoded and gzip_local: raise CommandException( 'Specifying both the -j/-J and -z/-Z options together is invalid.') if gzip_arg_exts and gzip_arg_all: if gzip_encoded: raise CommandException( 'Specifying both the -j and -J options together is invalid.') else: raise CommandException( 'Specifying both the -z and -Z options together is invalid.') self.gzip_exts = gzip_arg_exts or gzip_arg_all self.gzip_encoded = gzip_encoded return CreateCopyHelperOpts( perform_mv=perform_mv, no_clobber=no_clobber, daisy_chain=daisy_chain, read_args_from_stdin=read_args_from_stdin, print_ver=print_ver, use_manifest=use_manifest, preserve_acl=preserve_acl, canned_acl=canned_acl, skip_unsupported_objects=self.skip_unsupported_objects, test_callback_file=test_callback_file, dest_storage_class=dest_storage_class) perfdiag.py 0000644 00000272627 15025166704 0006725 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains the perfdiag gsutil command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar from collections import defaultdict from collections import namedtuple import contextlib import datetime import json import logging import math import multiprocessing import os import random import re import socket import string import subprocess import tempfile import time import boto import boto.gs.connection import six from six.moves import cStringIO from six.moves import http_client from six.moves import range from six.moves import range import gslib from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.command_argument import CommandArgument from gslib.commands import config from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.file_part import FilePart from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import ResumableThreshold from gslib.utils.constants import UTF8 from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.cloud_api_helper import GetDownloadSerializationData from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.system_util import CheckFreeSpace from gslib.utils.system_util import GetDiskCounters from gslib.utils.system_util import GetFileSize from gslib.utils.system_util import IS_LINUX from gslib.utils.system_util import IsRunningInCiEnvironment from gslib.utils.unit_util import DivideAndCeil from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeBitsHumanReadable from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import Percentile _SYNOPSIS = """ gsutil perfdiag [-i in.json] gsutil perfdiag [-o out.json] [-n objects] [-c processes] [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory] [-t tests] [-j ratio] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud Storage bucket. The ``bucket_name`` parameter must name an existing bucket to which the user has write permission. Several test files will be uploaded to and downloaded from this bucket. All test files will be deleted at the completion of the diagnostic if it finishes successfully. For a list of relevant permissions, see `Cloud IAM permissions for gsutil commands <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_. gsutil performance can be influenced by a number of factors originating at the client, server, or network level. Some examples include the following: + CPU speed + Available memory + The access path to the local disk + Network bandwidth + Contention and error rates along the path between gsutil and Google servers + Operating system buffering configuration + Firewalls and other network elements The `perfdiag` command is provided so that customers can run a known measurement suite when troubleshooting performance problems. <B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B> If the Cloud Storage team asks you to run a performance diagnostic please use the following command, and email the output file (output.json) to the @google.com address provided by the Cloud Storage team: gsutil perfdiag -o output.json gs://your-bucket Additional resources for discussing ``perfdiag`` results include the `Stack Overflow tag for Cloud Storage <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil/issues>`_. <B>OPTIONS</B> -n Sets the number of objects to use when downloading and uploading files during tests. Defaults to 5. -c Sets the number of `processes <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use while running throughput experiments. The default value is 1. -k Sets the number of `threads <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process to use while running throughput experiments. Each process will receive an equal number of threads. The default value is 1. NOTE: All specified threads and processes will be created, but may not by saturated with work if too few objects (specified with ``-n``) and too few components (specified with ``-y``) are specified. -p Sets the type of parallelism to be used (only applicable when threads or processes are specified and threads * processes > 1). The default is to use ``fan``. Must be one of the following: fan Use one thread per object. This is akin to using gsutil ``-m cp``, with sliced object download / parallel composite upload disabled. slice Use Y (specified with ``-y``) threads for each object, transferring one object at a time. This is akin to using parallel object download / parallel composite upload, without ``-m``. Sliced uploads not supported for s3. both Use Y (specified with ``-y``) threads for each object, transferring multiple objects at a time. This is akin to simultaneously using sliced object download / parallel composite upload and ``gsutil -m cp``. Parallel composite uploads not supported for s3. -y Sets the number of slices to divide each file/object into while transferring data. Only applicable with the slice (or both) parallelism type. The default is 4 slices. -s Sets the size (in bytes) for each of the N (set with ``-n``) objects used in the read and write throughput tests. The default is 1 MiB. This can also be specified using byte suffixes such as 500K or 1M. NOTE: these values are interpreted as multiples of 1024 (K=1024, M=1024*1024, etc.) NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with ``-n``) times as much disk space as specified will be required for the operation. -d Sets the directory to store temporary local files in. If not specified, a default temporary directory will be used. -t Sets the list of diagnostic tests to perform. The default is to run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a comma-separated list containing one or more of the following: lat For N (set with ``-n``) objects, write the object, retrieve its metadata, read the object, and finally delete the object. Record the latency of each operation. list Write N (set with ``-n``) objects to the bucket, record how long it takes for the eventually consistent listing call to return the N objects in its result, delete the N objects, then record how long it takes listing to stop returning the N objects. rthru Runs N (set with ``-n``) read operations, with at most C (set with -c) reads outstanding at any given time. rthru_file The same as ``rthru``, but simultaneously writes data to the disk, to gauge the performance impact of the local disk on downloads. wthru Runs N (set with ``-n``) write operations, with at most C (set with ``-c``) writes outstanding at any given time. wthru_file The same as wthru, but simultaneously reads data from the disk, to gauge the performance impact of the local disk on uploads. -m Adds metadata to the result JSON file. Multiple ``-m`` values can be specified. Example: gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname Each metadata key will be added to the top-level "metadata" dictionary in the output JSON file. -o Writes the results of the diagnostic to an output file. The output is a JSON file containing system information and performance diagnostic results. The file can be read and reported later using the ``-i`` option. -i Reads the JSON output file created using the ``-o`` command and prints a formatted description of the results. -j Applies gzip transport encoding and sets the target compression ratio for the generated test files. This ratio can be an integer between 0 and 100 (inclusive), with 0 generating a file with uniform data, and 100 generating random data. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. See `cp -j <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for specific semantics. <B>MEASURING AVAILABILITY</B> The ``perfdiag`` command ignores the boto num_retries configuration parameter. Instead, it always retries on HTTP errors in the 500 range and keeps track of how many 500 errors were encountered during the test. The availability measurement is reported at the end of the test. Note that HTTP responses are only recorded when the request was made in a single process. When using multiple processes or threads, read and write throughput measurements are performed in an external process, so the availability numbers reported won't include the throughput measurements. <B>NOTE</B> The ``perfdiag`` command runs a series of tests that collects system information, such as the following: + Retrieves requester's IP address. + Executes DNS queries to Google servers and collects the results. + Collects network statistics information from the output of ``netstat -s`` and evaluates the BIOS product name string. + If a proxy server is configured, attempts to connect to it to retrieve the location and storage class of the bucket being used for performance testing. None of this information will be sent to Google unless you proactively choose to send it. """) FileDataTuple = namedtuple('FileDataTuple', 'size md5 data') # Describes one object in a fanned download. If need_to_slice is specified as # True, the object should be downloaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Download. FanDownloadTuple = namedtuple( 'FanDownloadTuple', 'need_to_slice object_name file_name serialization_data') # Describes one slice in a sliced download. # Field names are the same as documented in PerfDiagCommand.Download. SliceDownloadTuple = namedtuple( 'SliceDownloadTuple', 'object_name file_name serialization_data start_byte end_byte') # Describes one file in a fanned upload. If need_to_slice is specified as # True, the file should be uploaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Upload. FanUploadTuple = namedtuple( 'FanUploadTuple', 'need_to_slice file_name object_name use_file gzip_encoded') # Describes one slice in a sliced upload. # Field names are the same as documented in PerfDiagCommand.Upload. SliceUploadTuple = namedtuple( 'SliceUploadTuple', 'file_name object_name use_file file_start file_size gzip_encoded') # Dict storing file_path:FileDataTuple for each temporary file used by # perfdiag. This data should be kept outside of the PerfDiagCommand class # since calls to Apply will make copies of all member data. temp_file_dict = {} class Error(Exception): """Base exception class for this module.""" pass class InvalidArgument(Error): """Raised on invalid arguments to functions.""" pass def _DownloadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel downloads. Args: cls: The calling PerfDiagCommand class instance. args: A FanDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedDownload(args.object_name, args.file_name, args.serialization_data) else: cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data) def _DownloadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced downloads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data, args.start_byte, args.end_byte) def _UploadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A FanUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedUpload(args.file_name, args.object_name, args.use_file, gsutil_api, gzip_encoded=args.gzip_encoded) else: cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, gzip_encoded=args.gzip_encoded) def _UploadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, args.file_start, args.file_size, gzip_encoded=args.gzip_encoded) def _DeleteWrapper(cls, object_name, thread_state=None): """Function argument to apply for performing parallel object deletions. Args: cls: The calling PerfDiagCommand class instance. object_name: The object name to delete from the test bucket. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Delete(object_name, gsutil_api) def _PerfdiagExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) def _DummyTrackerCallback(_): pass class DummyFile(object): """A dummy, file-like object that throws away everything written to it.""" # Because Python2 is so loose about bytes and text, Python3 code really # works best with a hint about a file object's mode. mode = 'bw' def write(self, *args, **kwargs): # pylint: disable=invalid-name pass def close(self): # pylint: disable=invalid-name pass def _GenerateFileData(fp, file_size=0, random_ratio=100, max_unique_random_bytes=5242883): """Writes data into a file like object. Args: fp: A file like object to write the data to. file_size: The amount of data to write to the file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. max_unique_random_bytes: The maximum number of bytes to generate pseudo-randomly before beginning to repeat bytes. The default was chosen as the next prime larger than 5 MiB. """ # Normalize the ratio. random_ratio /= 100.0 random_bytes = os.urandom(min(file_size, max_unique_random_bytes)) total_bytes_written = 0 while total_bytes_written < file_size: num_bytes = min(max_unique_random_bytes, file_size - total_bytes_written) # Calculate the amount of random and sequential data to write and # resolve rounding errors by adjusting the random bytes to write. num_bytes_seq = int(num_bytes * (1 - random_ratio)) num_bytes_random = num_bytes - num_bytes_seq fp.write(random_bytes[:num_bytes_random]) fp.write(b'x' * num_bytes_seq) total_bytes_written += num_bytes # Many functions in perfdiag re-define a temporary function based on a # variable from a loop, resulting in a false positive from the linter. # pylint: disable=cell-var-from-loop class PerfDiagCommand(Command): """Implementation of gsutil perfdiag command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'perfdiag', command_name_aliases=['diag', 'diagnostic', 'perf', 'performance'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='n:c:k:p:y:s:d:t:m:i:o:j:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudBucketURLsArgument(1)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='perfdiag', help_name_aliases=[], help_type='command_help', help_one_line_summary='Run performance diagnostic', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # Byte sizes to use for latency testing files. # TODO: Consider letting the user specify these sizes with a configuration # parameter. test_lat_file_sizes = ( 0, # 0 bytes 1024, # 1 KiB 102400, # 100 KiB 1048576, # 1 MiB ) # Test names. RTHRU = 'rthru' RTHRU_FILE = 'rthru_file' WTHRU = 'wthru' WTHRU_FILE = 'wthru_file' LAT = 'lat' LIST = 'list' # Parallelism strategies. FAN = 'fan' SLICE = 'slice' BOTH = 'both' # List of all diagnostic tests. ALL_DIAG_TESTS = (RTHRU, RTHRU_FILE, WTHRU, WTHRU_FILE, LAT, LIST) # List of diagnostic tests to run by default. DEFAULT_DIAG_TESTS = (RTHRU, WTHRU, LAT) # List of parallelism strategies. PARALLEL_STRATEGIES = (FAN, SLICE, BOTH) # Google Cloud Storage XML API endpoint host. XML_API_HOST = boto.config.get('Credentials', 'gs_host', boto.gs.connection.GSConnection.DefaultHost) # Google Cloud Storage XML API endpoint port. XML_API_PORT = boto.config.getint('Credentials', 'gs_port', 80) # Maximum number of times to retry requests on 5xx errors. MAX_SERVER_ERROR_RETRIES = 5 # Maximum number of times to retry requests on more serious errors like # the socket breaking. MAX_TOTAL_RETRIES = 10 # The default buffer size in boto's Key object is set to 8 KiB. This becomes a # bottleneck at high throughput rates, so we increase it. KEY_BUFFER_SIZE = 16384 # Maximum amount of time, in seconds, we will wait for object listings to # reflect what we expect in the listing tests. MAX_LISTING_WAIT_TIME = 60.0 def _Exec(self, cmd, raise_on_error=True, return_output=False, mute_stderr=False): """Executes a command in a subprocess. Args: cmd: List containing the command to execute. raise_on_error: Whether or not to raise an exception when a process exits with a non-zero return code. return_output: If set to True, the return value of the function is the stdout of the process. mute_stderr: If set to True, the stderr of the process is not printed to the console. Returns: The return code of the process or the stdout if return_output is set. Raises: Exception: If raise_on_error is set to True and any process exits with a non-zero return code. """ self.logger.debug('Running command: %s', cmd) stderr = subprocess.PIPE if mute_stderr else None p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr) (stdoutdata, _) = p.communicate() if six.PY3: if isinstance(stdoutdata, bytes): stdoutdata = stdoutdata.decode(UTF8) if raise_on_error and p.returncode: raise CommandException("Received non-zero return code (%d) from " "subprocess '%s'." % (p.returncode, ' '.join(cmd))) return stdoutdata if return_output else p.returncode def _WarnIfLargeData(self): """Outputs a warning message if a large amount of data is being used.""" if self.num_objects * self.thru_filesize > HumanReadableToBytes('2GiB'): self.logger.info('This is a large operation, and could take a while.') def _MakeTempFile(self, file_size=0, mem_metadata=False, mem_data=False, prefix='gsutil_test_file', random_ratio=100): """Creates a temporary file of the given size and returns its path. Args: file_size: The size of the temporary file to create. mem_metadata: If true, store md5 and file size in memory at temp_file_dict[fpath].md5, tempfile_data[fpath].file_size. mem_data: If true, store the file data in memory at temp_file_dict[fpath].data prefix: The prefix to use for the temporary file. Defaults to gsutil_test_file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. Returns: The file path of the created temporary file. """ fd, fpath = tempfile.mkstemp(suffix='.bin', prefix=prefix, dir=self.directory, text=False) with os.fdopen(fd, 'wb') as fp: _GenerateFileData(fp, file_size, random_ratio) if mem_metadata or mem_data: with open(fpath, 'rb') as fp: file_size = GetFileSize(fp) if mem_metadata else None md5 = CalculateB64EncodedMd5FromContents(fp) if mem_metadata else None data = fp.read() if mem_data else None temp_file_dict[fpath] = FileDataTuple(file_size, md5, data) self.temporary_files.add(fpath) return fpath def _SetUp(self): """Performs setup operations needed before diagnostics can be run.""" # Stores test result data. self.results = {} # Set of file paths for local temporary files. self.temporary_files = set() # Set of names for test objects that exist in the test bucket. self.temporary_objects = set() # Total number of HTTP requests made. self.total_requests = 0 # Total number of HTTP 5xx errors. self.request_errors = 0 # Number of responses, keyed by response code. self.error_responses_by_code = defaultdict(int) # Total number of socket errors. self.connection_breaks = 0 # Boolean to prevent doing cleanup twice. self.teardown_completed = False # Create files for latency test. if self.LAT in self.diag_tests: self.latency_files = [] for file_size in self.test_lat_file_sizes: fpath = self._MakeTempFile(file_size, mem_metadata=True, mem_data=True) self.latency_files.append(fpath) # Create files for throughput tests. if self.diag_tests.intersection( (self.RTHRU, self.WTHRU, self.RTHRU_FILE, self.WTHRU_FILE)): # Create a file for warming up the TCP connection. self.tcp_warmup_file = self._MakeTempFile(5 * 1024 * 1024, mem_metadata=True, mem_data=True) # For in memory tests, throughput tests transfer the same object N times # instead of creating N objects, in order to avoid excessive memory usage. if self.diag_tests.intersection((self.RTHRU, self.WTHRU)): self.mem_thru_file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, mem_data=True, random_ratio=self.gzip_compression_ratio) self.mem_thru_object_name = os.path.basename(self.mem_thru_file_name) # For tests that use disk I/O, it is necessary to create N objects in # in order to properly measure the performance impact of seeks. if self.diag_tests.intersection((self.RTHRU_FILE, self.WTHRU_FILE)): # List of file names and corresponding object names to use for file # throughput tests. self.thru_file_names = [] self.thru_object_names = [] free_disk_space = CheckFreeSpace(self.directory) if free_disk_space >= self.thru_filesize * self.num_objects: self.logger.info( '\nCreating %d local files each of size %s.' % (self.num_objects, MakeHumanReadable(self.thru_filesize))) self._WarnIfLargeData() for _ in range(self.num_objects): file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, random_ratio=self.gzip_compression_ratio) self.thru_file_names.append(file_name) self.thru_object_names.append(os.path.basename(file_name)) else: raise CommandException( 'Not enough free disk space for throughput files: ' '%s of disk space required, but only %s available.' % (MakeHumanReadable(self.thru_filesize * self.num_objects), MakeHumanReadable(free_disk_space))) # Dummy file buffer to use for downloading that goes nowhere. self.discard_sink = DummyFile() # Filter out misleading progress callback output and the incorrect # suggestion to use gsutil -m perfdiag. self.logger.addFilter(self._PerfdiagFilter()) def _TearDown(self): """Performs operations to clean things up after performing diagnostics.""" if not self.teardown_completed: temp_file_dict.clear() try: for fpath in self.temporary_files: os.remove(fpath) if self.delete_directory: os.rmdir(self.directory) except OSError: pass # I usually hate leaving commented out code in commits, but this is a # rare exception. This commented out code attempted to clean up the # files created by the perfdiag command. If there is only one process and # thread, it worked by serializing the deletes. If it had more than one # process (multiple threads on a single process seemed to work fine, but # don't quote me on that), it tried to fire up a multi-processing delete # that used the existing process pool created during the previous part of # the perfdiag command execution. This randomly fails in Python 3. So I # have left this code here, commented out, and have replaced it with the # sequential delete part only. That this fails in Python 3 will probably # present as other problems later. At least a multiprocessing rm using # gsutil -m appears to always work, so it is something to do with this # usage of parallel override and existing process pools, I think. # PS - only fails when using prefer XML. # if self.threads > 1 or self.processes > 1: # args = [obj for obj in self.temporary_objects] # self.Apply( # _DeleteWrapper, args, _PerfdiagExceptionHandler, # arg_checker=DummyArgChecker, # parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, # process_count=self.processes, thread_count=self.threads) # else: # for object_name in self.temporary_objects: # self.Delete(object_name, self.gsutil_api) for object_name in self.temporary_objects: self.Delete(object_name, self.gsutil_api) self.teardown_completed = True @contextlib.contextmanager def _Time(self, key, bucket): """A context manager that measures time. A context manager that prints a status message before and after executing the inner command and times how long the inner command takes. Keeps track of the timing, aggregated by the given key. Args: key: The key to insert the timing value into a dictionary bucket. bucket: A dictionary to place the timing value in. Yields: For the context manager. """ self.logger.info('%s starting...', key) t0 = time.time() yield t1 = time.time() bucket[key].append(t1 - t0) self.logger.info('%s done.', key) def _RunOperation(self, func): """Runs an operation with retry logic. Args: func: The function to run. Returns: True if the operation succeeds, False if aborted. """ # We retry on httplib exceptions that can happen if the socket was closed # by the remote party or the connection broke because of network issues. # Only the BotoServerError is counted as a 5xx error towards the retry # limit. success = False server_error_retried = 0 total_retried = 0 i = 0 return_val = None while not success: next_sleep = min(random.random() * (2**i) + 1, GetMaxRetryDelay()) try: return_val = func() self.total_requests += 1 success = True except tuple(self.exceptions) as e: total_retried += 1 if total_retried > self.MAX_TOTAL_RETRIES: self.logger.info('Reached maximum total retries. Not retrying.') break if isinstance(e, ServiceException): if e.status >= 500: self.error_responses_by_code[e.status] += 1 self.total_requests += 1 self.request_errors += 1 server_error_retried += 1 time.sleep(next_sleep) else: raise if server_error_retried > self.MAX_SERVER_ERROR_RETRIES: self.logger.info( 'Reached maximum server error retries. Not retrying.') break else: self.connection_breaks += 1 return return_val def _RunLatencyTests(self): """Runs latency tests.""" # Stores timing information for each category of operation. self.results['latency'] = defaultdict(list) for i in range(self.num_objects): self.logger.info('\nRunning latency iteration %d...', i + 1) for fpath in self.latency_files: file_data = temp_file_dict[fpath] url = self.bucket_url.Clone() url.object_name = os.path.basename(fpath) file_size = file_data.size readable_file_size = MakeHumanReadable(file_size) self.logger.info( "\nFile of size %s located on disk at '%s' being diagnosed in the " "cloud at '%s'.", readable_file_size, fpath, url) upload_target = StorageUrlToUploadObjectMetadata(url) def _Upload(): io_fp = six.BytesIO(file_data.data) with self._Time('UPLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.UploadObject(io_fp, upload_target, size=file_size, provider=self.provider, fields=['name']) self._RunOperation(_Upload) def _Metadata(): with self._Time('METADATA_%d' % file_size, self.results['latency']): return self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=self.provider, fields=['name', 'contentType', 'mediaLink', 'size']) # Download will get the metadata first if we don't pass it in. download_metadata = self._RunOperation(_Metadata) serialization_data = GetDownloadSerializationData(download_metadata) def _Download(): with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.GetObjectMedia( url.bucket_name, url.object_name, self.discard_sink, provider=self.provider, serialization_data=serialization_data) self._RunOperation(_Download) def _Delete(): with self._Time('DELETE_%d' % file_size, self.results['latency']): self.gsutil_api.DeleteObject(url.bucket_name, url.object_name, provider=self.provider) self._RunOperation(_Delete) class _PerfdiagFilter(logging.Filter): def filter(self, record): # Used to prevent unnecessary output when using multiprocessing. msg = record.getMessage() return not (('Copying file:///' in msg) or ('Copying gs://' in msg) or ('Computing CRC' in msg) or ('gsutil -m perfdiag' in msg)) def _PerfdiagExceptionHandler(self, e): """Simple exception handler to allow post-completion status.""" self.logger.error(str(e)) def PerformFannedDownload(self, need_to_slice, object_names, file_names, serialization_data): """Performs a parallel download of multiple objects using the fan strategy. Args: need_to_slice: If True, additionally apply the slice strategy to each object in object_names. object_names: A list of object names to be downloaded. Each object must already exist in the test bucket. file_names: A list, corresponding by index to object_names, of file names for downloaded data. If None, discard downloaded data. serialization_data: A list, corresponding by index to object_names, of serialization data for each object. """ args = [] for i in range(len(object_names)): file_name = file_names[i] if file_names else None args.append( FanDownloadTuple(need_to_slice, object_names[i], file_name, serialization_data[i])) self.Apply( _DownloadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedDownload(self, object_name, file_name, serialization_data): """Performs a download of an object using the slice strategy. Args: object_name: The name of the object to download. file_name: The name of the file to download data to, or None if data should be discarded. serialization_data: The serialization data for the object. """ if file_name: with open(file_name, 'ab') as fp: fp.truncate(self.thru_filesize) component_size = DivideAndCeil(self.thru_filesize, self.num_slices) args = [] for i in range(self.num_slices): start_byte = i * component_size end_byte = min((i + 1) * (component_size) - 1, self.thru_filesize - 1) args.append( SliceDownloadTuple(object_name, file_name, serialization_data, start_byte, end_byte)) self.Apply( _DownloadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformFannedUpload(self, need_to_slice, file_names, object_names, use_file, gzip_encoded=False): """Performs a parallel upload of multiple files using the fan strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data for file_name should be present in temp_file_dict if use_file is specified as False. Args: need_to_slice: If True, additionally apply the slice strategy to each file in file_names. file_names: A list of file names to be uploaded. object_names: A list, corresponding by by index to file_names, of object names to upload data to. use_file: If true, use disk I/O, otherwise read upload data from memory. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. """ args = [] for i in range(len(file_names)): args.append( FanUploadTuple(need_to_slice, file_names[i], object_names[i], use_file, gzip_encoded)) self.Apply( _UploadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedUpload(self, file_name, object_name, use_file, gsutil_api, gzip_encoded=False): """Performs a parallel upload of a file using the slice strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data from for file_name should be present in temp_file_dict if use_file is specified as False. Args: file_name: The name of the file to upload. object_name: The name of the object to upload to. use_file: If true, use disk I/O, otherwise read upload data from memory. gsutil_api: CloudApi instance to use for operations in this thread. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a semaphore is used to limit resource usage. """ # Divide the file into components. component_size = DivideAndCeil(self.thru_filesize, self.num_slices) component_object_names = ([ object_name + str(i) for i in range(self.num_slices) ]) args = [] for i in range(self.num_slices): component_start = i * component_size component_size = min(component_size, temp_file_dict[file_name].size - component_start) args.append( SliceUploadTuple(file_name, component_object_names[i], use_file, component_start, component_size, gzip_encoded)) # Upload the components in parallel. try: self.Apply( _UploadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) # Compose the components into an object. request_components = [] for i in range(self.num_slices): src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=component_object_names[i])) request_components.append(src_obj_metadata) def _Compose(): dst_obj_metadata = apitools_messages.Object() dst_obj_metadata.name = object_name dst_obj_metadata.bucket = self.bucket_url.bucket_name gsutil_api.ComposeObject(request_components, dst_obj_metadata, provider=self.provider) self._RunOperation(_Compose) finally: # Delete the temporary components. self.Apply( _DeleteWrapper, component_object_names, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def _RunReadThruTests(self, use_file=False): """Runs read throughput tests.""" test_name = 'read_throughput_file' if use_file else 'read_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning read throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy } # Copy the file(s) to the test bucket, and also get the serialization data # so that we can pass it to download. if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names serialization_data = [] for i in range(self.num_objects): self.temporary_objects.add(self.thru_object_names[i]) if self.WTHRU_FILE in self.diag_tests: # If we ran the WTHRU_FILE test, then the objects already exist. obj_metadata = self.gsutil_api.GetObjectMetadata( self.bucket_url.bucket_name, self.thru_object_names[i], fields=['size', 'mediaLink'], provider=self.bucket_url.scheme) else: obj_metadata = self.Upload(self.thru_file_names[i], self.thru_object_names[i], self.gsutil_api, use_file) # File overwrite causes performance issues with sliced downloads. # Delete the file and reopen it for download. This matches what a real # download would look like. os.unlink(self.thru_file_names[i]) open(self.thru_file_names[i], 'ab').close() serialization_data.append(GetDownloadSerializationData(obj_metadata)) else: # For in-memory test only use one file but copy it num_objects times, to # allow scalability in num_objects. self.temporary_objects.add(self.mem_thru_object_name) obj_metadata = self.Upload(self.mem_thru_file_name, self.mem_thru_object_name, self.gsutil_api, use_file) file_names = None object_names = [self.mem_thru_object_name] * self.num_objects serialization_data = ([GetDownloadSerializationData(obj_metadata)] * self.num_objects) # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) self.Download(warmup_obj_name, self.gsutil_api) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.Download(object_names[i], self.gsutil_api, file_name, serialization_data[i]) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedDownload(need_to_slice, object_names, file_names, serialization_data) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.PerformSlicedDownload(object_names[i], file_name, serialization_data[i]) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunWriteThruTests(self, use_file=False): """Runs write throughput tests.""" test_name = 'write_throughput_file' if use_file else 'write_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning write throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy, 'gzip_encoded_writes': self.gzip_encoded_writes, 'gzip_compression_ratio': self.gzip_compression_ratio } # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names else: # For in-memory test only use one file but copy it num_objects times, to # allow for scalability in num_objects. file_names = [self.mem_thru_file_name] * self.num_objects object_names = ([ self.mem_thru_object_name + str(i) for i in range(self.num_objects) ]) for object_name in object_names: self.temporary_objects.add(object_name) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): self.Upload(file_names[i], object_names[i], self.gsutil_api, use_file, gzip_encoded=self.gzip_encoded_writes) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedUpload(need_to_slice, file_names, object_names, use_file, gzip_encoded=self.gzip_encoded_writes) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): self.PerformSlicedUpload(file_names[i], object_names[i], use_file, self.gsutil_api, gzip_encoded=self.gzip_encoded_writes) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunListTests(self): """Runs eventual consistency listing latency tests.""" self.results['listing'] = {'num_files': self.num_objects} # Generate N random objects to put into the bucket. list_objects = [] args = [] # Differentiate objects created by each perfdiag execution so that leftovers # from a previous run (if perfdiag could not exit gracefully and delete # them) do not affect this run. random_id = ''.join( [random.choice(string.ascii_lowercase) for _ in range(10)]) list_prefix = 'gsutil-perfdiag-list-' + random_id + '-' for _ in range(self.num_objects): fpath = self._MakeTempFile(0, mem_data=True, mem_metadata=True, prefix=list_prefix) object_name = os.path.basename(fpath) list_objects.append(object_name) args.append(FanUploadTuple(False, fpath, object_name, False, False)) self.temporary_objects.add(object_name) # Add the objects to the bucket. self.logger.info('\nWriting %s objects for listing test...', self.num_objects) self.Apply(_UploadObject, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) list_latencies = [] files_seen = [] total_start_time = time.time() expected_objects = set(list_objects) found_objects = set() def _List(): """Lists and returns objects in the bucket. Also records latency.""" t0 = time.time() objects = list( self.gsutil_api.ListObjects(self.bucket_url.bucket_name, prefix=list_prefix, delimiter='/', provider=self.provider, fields=['items/name'])) if len(objects) > self.num_objects: self.logger.warning( 'Listing produced more than the expected %d object(s).', self.num_objects) t1 = time.time() list_latencies.append(t1 - t0) return set([obj.data.name for obj in objects]) def _ListAfterUpload(): names = _List() found_objects.update(names & expected_objects) files_seen.append(len(found_objects)) self.logger.info('Listing bucket %s waiting for %s objects to appear...', self.bucket_url.bucket_name, self.num_objects) while expected_objects - found_objects: self._RunOperation(_ListAfterUpload) if expected_objects - found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['insert'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } args = [object_name for object_name in list_objects] self.logger.info('Deleting %s objects for listing test...', self.num_objects) self.Apply(_DeleteWrapper, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) self.logger.info('Listing bucket %s waiting for %s objects to disappear...', self.bucket_url.bucket_name, self.num_objects) list_latencies = [] files_seen = [] total_start_time = time.time() found_objects = set(list_objects) while found_objects: def _ListAfterDelete(): names = _List() found_objects.intersection_update(names) files_seen.append(len(found_objects)) self._RunOperation(_ListAfterDelete) if found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['delete'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } def Upload(self, file_name, object_name, gsutil_api, use_file=False, file_start=0, file_size=None, gzip_encoded=False): """Performs an upload to the test bucket. The file is uploaded to the bucket referred to by self.bucket_url, and has name object_name. Args: file_name: The path to the local file, and the key to its entry in temp_file_dict. object_name: The name of the remote object. gsutil_api: Cloud API instance to use for this upload. use_file: If true, use disk I/O, otherwise read everything from memory. file_start: The first byte in the file to upload to the object. (only should be specified for sliced uploads) file_size: The size of the file to upload. (only should be specified for sliced uploads) gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. Returns: Uploaded Object Metadata. """ fp = None if file_size is None: file_size = temp_file_dict[file_name].size upload_url = self.bucket_url.Clone() upload_url.object_name = object_name upload_target = StorageUrlToUploadObjectMetadata(upload_url) try: if use_file: fp = FilePart(file_name, file_start, file_size) else: data = temp_file_dict[file_name].data[file_start:file_start + file_size] fp = six.BytesIO(data) def _InnerUpload(): if file_size < ResumableThreshold(): return gsutil_api.UploadObject(fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], gzip_encoded=gzip_encoded) else: return gsutil_api.UploadObjectResumable( fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], tracker_callback=_DummyTrackerCallback, gzip_encoded=gzip_encoded) # Because perfdiag does not go through copy_helper, it misses the # resource lock. Here, if the upload is compressed, the lock is applied # manually. def _InnerUploadLock(): with gslib.command.concurrent_compressed_upload_lock: return _InnerUpload() upload_func = _InnerUploadLock if gzip_encoded else _InnerUpload return self._RunOperation(upload_func) finally: if fp: fp.close() def Download(self, object_name, gsutil_api, file_name=None, serialization_data=None, start_byte=0, end_byte=None): """Downloads an object from the test bucket. Args: object_name: The name of the object (in the test bucket) to download. gsutil_api: CloudApi instance to use for the download. file_name: Optional file name to write downloaded data to. If None, downloaded data is discarded immediately. serialization_data: Optional serialization data, used so that we don't have to get the metadata before downloading. start_byte: The first byte in the object to download. (only should be specified for sliced downloads) end_byte: The last byte in the object to download. (only should be specified for sliced downloads) """ fp = None try: if file_name is not None: fp = open(file_name, 'r+b') fp.seek(start_byte) else: fp = self.discard_sink def _InnerDownload(): gsutil_api.GetObjectMedia(self.bucket_url.bucket_name, object_name, fp, provider=self.provider, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data) self._RunOperation(_InnerDownload) finally: if fp: fp.close() def Delete(self, object_name, gsutil_api): """Deletes an object from the test bucket. Args: object_name: The name of the object to delete. gsutil_api: Cloud API instance to use for this delete. """ try: def _InnerDelete(): gsutil_api.DeleteObject(self.bucket_url.bucket_name, object_name, provider=self.provider) self._RunOperation(_InnerDelete) except NotFoundException: pass def _GetDiskCounters(self): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = [] with open('/proc/partitions', 'r') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) retdict = {} with open('/proc/diskstats', 'r') as f: for line in f: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) return retdict def _GetTcpStats(self): """Tries to parse out TCP packet information from netstat output. Returns: A dictionary containing TCP information, or None if netstat is not available. """ # netstat return code is non-zero for -s on Linux, so don't raise on error. try: netstat_output = self._Exec(['netstat', '-s'], return_output=True, raise_on_error=False) except OSError: self.logger.warning('netstat not found on your system; some measurement ' 'data will be missing') return None netstat_output = netstat_output.strip().lower() found_tcp = False tcp_retransmit = None tcp_received = None tcp_sent = None for line in netstat_output.split('\n'): # Header for TCP section is "Tcp:" in Linux/Mac and # "TCP Statistics for" in Windows. if 'tcp:' in line or 'tcp statistics' in line: found_tcp = True # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts" # Windows == "segments retransmitted". if (found_tcp and tcp_retransmit is None and ('segments retransmited' in line or 'retransmit timeouts' in line or 'segments retransmitted' in line)): tcp_retransmit = ''.join(c for c in line if c in string.digits) # Linux+Windows == "segments received", Mac == "packets received". if (found_tcp and tcp_received is None and ('segments received' in line or 'packets received' in line)): tcp_received = ''.join(c for c in line if c in string.digits) # Linux == "segments send out" (sic), Mac+Windows == "packets sent". if (found_tcp and tcp_sent is None and ('segments send out' in line or 'packets sent' in line or 'segments sent' in line)): tcp_sent = ''.join(c for c in line if c in string.digits) result = {} try: result['tcp_retransmit'] = int(tcp_retransmit) result['tcp_received'] = int(tcp_received) result['tcp_sent'] = int(tcp_sent) except (ValueError, TypeError): result['tcp_retransmit'] = None result['tcp_received'] = None result['tcp_sent'] = None return result def _CollectSysInfo(self): """Collects system information.""" sysinfo = {} # All exceptions that might be raised from socket module calls. socket_errors = (socket.error, socket.herror, socket.gaierror, socket.timeout) # Find out whether HTTPS is enabled in Boto. sysinfo['boto_https_enabled'] = boto.config.get('Boto', 'is_secure', True) # Look up proxy info. proxy_host = boto.config.get('Boto', 'proxy', None) proxy_port = boto.config.getint('Boto', 'proxy_port', 0) sysinfo['using_proxy'] = bool(proxy_host) if boto.config.get('Boto', 'proxy_rdns', True if proxy_host else False): self.logger.info( 'DNS lookups are disallowed in this environment, so some information ' 'is not included in this perfdiag run. To allow local DNS lookups ' 'while using a proxy, set proxy_rdns to False in your boto file.') # Get the local IP address from socket lib. try: sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname()) except socket_errors: sysinfo['ip_address'] = '' # Record the temporary directory used since it can affect performance, e.g. # when on a networked filesystem. sysinfo['tempdir'] = self.directory # Produces an RFC 2822 compliant GMT timestamp. sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()) # Execute a CNAME lookup on Google DNS to find what Google server # it's routing to. cmd = ['nslookup', '-type=CNAME', self.XML_API_HOST] try: nslookup_cname_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output) sysinfo['googserv_route'] = m.group('googserv') if m else None except (CommandException, OSError): sysinfo['googserv_route'] = '' # Check the BIOS product name, to determine if we're running on a GCE # instance. # TODO: Add support for checking if we're on GCE from Windows, via WMI. if IS_LINUX: try: with open('/sys/class/dmi/id/product_name', 'r') as fp: sysinfo['on_gce'] = (fp.readline() == 'Google Compute Engine\n') except OSError: pass # If we're running on GCE include details about the instance. if sysinfo.get('on_gce', False): hostname = socket.gethostname() cmd = ['gcloud', 'compute', 'instances', 'list', '--filter=', hostname] try: # "gcloud compute" commands will (hopefully) fail on these envs due to # lack of credentials/permissions to access compute resources. mute_stderr = IsRunningInCiEnvironment() sysinfo['gce_instance_info'] = (self._Exec(cmd, return_output=True, mute_stderr=mute_stderr)) except (CommandException, OSError): sysinfo['gce_instance_info'] = '' # Record info about location and storage class of the bucket being used for # performance testing. bucket_info = self.gsutil_api.GetBucket(self.bucket_url.bucket_name, fields=['location', 'storageClass'], provider=self.bucket_url.scheme) sysinfo['bucket_location'] = bucket_info.location sysinfo['bucket_storageClass'] = bucket_info.storageClass # Try to determine the latency of a DNS lookup for the Google hostname # endpoint. Note: we don't piggyback on gethostbyname_ex below because # the _ex version requires an extra RTT. try: t0 = time.time() socket.gethostbyname(self.XML_API_HOST) t1 = time.time() sysinfo['google_host_dns_latency'] = t1 - t0 except socket_errors: pass # Look up IP addresses for Google Server. try: (hostname, _, ipaddrlist) = socket.gethostbyname_ex(self.XML_API_HOST) sysinfo['googserv_ips'] = ipaddrlist except socket_errors: ipaddrlist = [] sysinfo['googserv_ips'] = [] # Reverse lookup the hostnames for the Google Server IPs. sysinfo['googserv_hostnames'] = [] for googserv_ip in ipaddrlist: try: (hostname, _, ipaddrlist) = socket.gethostbyaddr(googserv_ip) sysinfo['googserv_hostnames'].append(hostname) except socket_errors: pass # Query o-o to find out what the Google DNS thinks is the user's IP. try: cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.'] nslookup_txt_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output) sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None except (CommandException, OSError): sysinfo['dns_o-o_ip'] = '' # Try to determine the latency of connecting to the Google hostname # endpoint. sysinfo['google_host_connect_latencies'] = {} for googserv_ip in ipaddrlist: try: sock = socket.socket() t0 = time.time() sock.connect((googserv_ip, self.XML_API_PORT)) t1 = time.time() sysinfo['google_host_connect_latencies'][googserv_ip] = t1 - t0 except socket_errors: pass # If using a proxy, try to determine the latency of a DNS lookup to resolve # the proxy hostname and the latency of connecting to the proxy. if proxy_host: proxy_ip = None try: t0 = time.time() proxy_ip = socket.gethostbyname(proxy_host) t1 = time.time() sysinfo['proxy_dns_latency'] = t1 - t0 except socket_errors: pass try: sock = socket.socket() t0 = time.time() sock.connect((proxy_ip or proxy_host, proxy_port)) t1 = time.time() sysinfo['proxy_host_connect_latency'] = t1 - t0 except socket_errors: pass # Try and find the number of CPUs in the system if available. try: sysinfo['cpu_count'] = multiprocessing.cpu_count() except NotImplementedError: sysinfo['cpu_count'] = None # For *nix platforms, obtain the CPU load. try: sysinfo['load_avg'] = list(os.getloadavg()) except (AttributeError, OSError): sysinfo['load_avg'] = None # Try and collect memory information from /proc/meminfo if possible. mem_total = None mem_free = None mem_buffers = None mem_cached = None try: with open('/proc/meminfo', 'r') as f: for line in f: if line.startswith('MemTotal'): mem_total = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('MemFree'): mem_free = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Buffers'): mem_buffers = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Cached'): mem_cached = (int(''.join(c for c in line if c in string.digits)) * 1000) except (IOError, ValueError): pass sysinfo['meminfo'] = { 'mem_total': mem_total, 'mem_free': mem_free, 'mem_buffers': mem_buffers, 'mem_cached': mem_cached } # Get configuration attributes from config module. sysinfo['gsutil_config'] = {} for attr in dir(config): attr_value = getattr(config, attr) # Filter out multiline strings that are not useful. if attr.isupper() and not (isinstance(attr_value, six.string_types) and '\n' in attr_value): sysinfo['gsutil_config'][attr] = attr_value sysinfo['tcp_proc_values'] = {} stats_to_check = [ '/proc/sys/net/core/rmem_default', '/proc/sys/net/core/rmem_max', '/proc/sys/net/core/wmem_default', '/proc/sys/net/core/wmem_max', '/proc/sys/net/ipv4/tcp_timestamps', '/proc/sys/net/ipv4/tcp_sack', '/proc/sys/net/ipv4/tcp_window_scaling', ] for fname in stats_to_check: try: with open(fname, 'r') as f: value = f.read() sysinfo['tcp_proc_values'][os.path.basename(fname)] = value.strip() except IOError: pass self.results['sysinfo'] = sysinfo def _DisplayStats(self, trials): """Prints out mean, standard deviation, median, and 90th percentile.""" n = len(trials) mean = float(sum(trials)) / n stdev = math.sqrt(sum((x - mean)**2 for x in trials) / n) text_util.print_to_fd(str(n).rjust(6), '', end=' ') text_util.print_to_fd(('%.1f' % (mean * 1000)).rjust(9), '', end=' ') text_util.print_to_fd(('%.1f' % (stdev * 1000)).rjust(12), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), '') def _DisplayResults(self): """Displays results collected from diagnostic run.""" text_util.print_to_fd() text_util.print_to_fd('=' * 78) text_util.print_to_fd('DIAGNOSTIC RESULTS'.center(78)) text_util.print_to_fd('=' * 78) if 'latency' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Latency'.center(78)) text_util.print_to_fd('-' * 78) text_util.print_to_fd( 'Operation Size Trials Mean (ms) Std Dev (ms) ' 'Median (ms) 90th % (ms)') text_util.print_to_fd( '========= ========= ====== ========= ============ ' '=========== ===========') for key in sorted(self.results['latency']): trials = sorted(self.results['latency'][key]) op, numbytes = key.split('_') numbytes = int(numbytes) if op == 'METADATA': text_util.print_to_fd('Metadata'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DOWNLOAD': text_util.print_to_fd('Download'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'UPLOAD': text_util.print_to_fd('Upload'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DELETE': text_util.print_to_fd('Delete'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if 'write_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput'.center(78)) text_util.print_to_fd('-' * 78) write_thru = self.results['write_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru['file_size']), MakeHumanReadable(write_thru['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))) if 'parallelism' in write_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru['parallelism']) if 'write_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) write_thru_file = self.results['write_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru_file['file_size']), MakeHumanReadable(write_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru_file['bytes_per_second'] * 8))) if 'parallelism' in write_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru_file['parallelism']) if 'read_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput'.center(78)) text_util.print_to_fd('-' * 78) read_thru = self.results['read_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru['file_size']), MakeHumanReadable(read_thru['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))) if 'parallelism' in read_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru['parallelism']) if 'read_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) read_thru_file = self.results['read_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru_file['file_size']), MakeHumanReadable(read_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru_file['bytes_per_second'] * 8))) if 'parallelism' in read_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru_file['parallelism']) if 'listing' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Listing'.center(78)) text_util.print_to_fd('-' * 78) listing = self.results['listing'] insert = listing['insert'] delete = listing['delete'] text_util.print_to_fd('After inserting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % insert['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % insert['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in insert['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, insert['files_seen_after_listing'])))) text_util.print_to_fd('After deleting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % delete['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % delete['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in delete['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, delete['files_seen_after_listing'])))) if 'sysinfo' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('System Information'.center(78)) text_util.print_to_fd('-' * 78) info = self.results['sysinfo'] text_util.print_to_fd('IP Address: \n %s' % info['ip_address']) text_util.print_to_fd('Temporary Directory: \n %s' % info['tempdir']) text_util.print_to_fd('Bucket URI: \n %s' % self.results['bucket_uri']) text_util.print_to_fd('gsutil Version: \n %s' % self.results.get('gsutil_version', 'Unknown')) text_util.print_to_fd('boto Version: \n %s' % self.results.get('boto_version', 'Unknown')) if 'gmt_timestamp' in info: ts_string = info['gmt_timestamp'] timetuple = None try: # Convert RFC 2822 string to Linux timestamp. timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000') except ValueError: pass if timetuple: # Converts the GMT time tuple to local Linux timestamp. localtime = calendar.timegm(timetuple) localdt = datetime.datetime.fromtimestamp(localtime) text_util.print_to_fd('Measurement time: \n %s' % localdt.strftime('%Y-%m-%d %I:%M:%S %p %Z')) if 'on_gce' in info: text_util.print_to_fd('Running on GCE: \n %s' % info['on_gce']) if info['on_gce']: text_util.print_to_fd('GCE Instance:\n\t%s' % info['gce_instance_info'].replace('\n', '\n\t')) text_util.print_to_fd('Bucket location: \n %s' % info['bucket_location']) text_util.print_to_fd('Bucket storage class: \n %s' % info['bucket_storageClass']) text_util.print_to_fd('Google Server: \n %s' % info['googserv_route']) text_util.print_to_fd('Google Server IP Addresses: \n %s' % ('\n '.join(info['googserv_ips']))) text_util.print_to_fd('Google Server Hostnames: \n %s' % ('\n '.join(info['googserv_hostnames']))) text_util.print_to_fd('Google DNS thinks your IP is: \n %s' % info['dns_o-o_ip']) text_util.print_to_fd('CPU Count: \n %s' % info['cpu_count']) text_util.print_to_fd('CPU Load Average: \n %s' % info['load_avg']) try: text_util.print_to_fd(('Total Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_total']))) # Free memory is really MemFree + Buffers + Cached. text_util.print_to_fd('Free Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_free'] + info['meminfo']['mem_buffers'] + info['meminfo']['mem_cached'])) except TypeError: pass if 'netstat_end' in info and 'netstat_start' in info: netstat_after = info['netstat_end'] netstat_before = info['netstat_start'] for tcp_type in ('sent', 'received', 'retransmit'): try: delta = (netstat_after['tcp_%s' % tcp_type] - netstat_before['tcp_%s' % tcp_type]) text_util.print_to_fd('TCP segments %s during test:\n %d' % (tcp_type, delta)) except TypeError: pass else: text_util.print_to_fd( 'TCP segment counts not available because "netstat" was not ' 'found during test runs') if 'disk_counters_end' in info and 'disk_counters_start' in info: text_util.print_to_fd('Disk Counter Deltas:\n', end=' ') disk_after = info['disk_counters_end'] disk_before = info['disk_counters_start'] text_util.print_to_fd('', 'disk'.rjust(6), end=' ') for colname in [ 'reads', 'writes', 'rbytes', 'wbytes', 'rtime', 'wtime' ]: text_util.print_to_fd(colname.rjust(8), end=' ') text_util.print_to_fd() for diskname in sorted(disk_after): before = disk_before[diskname] after = disk_after[diskname] (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after text_util.print_to_fd('', diskname.rjust(6), end=' ') deltas = [ reads2 - reads1, writes2 - writes1, rbytes2 - rbytes1, wbytes2 - wbytes1, rtime2 - rtime1, wtime2 - wtime1 ] for delta in deltas: text_util.print_to_fd(str(delta).rjust(8), end=' ') text_util.print_to_fd() if 'tcp_proc_values' in info: text_util.print_to_fd('TCP /proc values:\n', end=' ') for item in six.iteritems(info['tcp_proc_values']): text_util.print_to_fd(' %s = %s' % item) if 'boto_https_enabled' in info: text_util.print_to_fd('Boto HTTPS Enabled: \n %s' % info['boto_https_enabled']) if 'using_proxy' in info: text_util.print_to_fd('Requests routed through proxy: \n %s' % info['using_proxy']) if 'google_host_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for Google Storage server (ms): ' '\n %.1f' % (info['google_host_dns_latency'] * 1000.0))) if 'google_host_connect_latencies' in info: text_util.print_to_fd( 'Latencies connecting to Google Storage server IPs (ms):') for ip, latency in six.iteritems(info['google_host_connect_latencies']): text_util.print_to_fd(' %s = %.1f' % (ip, latency * 1000.0)) if 'proxy_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for the configured proxy (ms): ' '\n %.1f' % (info['proxy_dns_latency'] * 1000.0))) if 'proxy_host_connect_latency' in info: text_util.print_to_fd( ('Latency connecting to the configured proxy (ms): \n %.1f' % (info['proxy_host_connect_latency'] * 1000.0))) if 'request_errors' in self.results and 'total_requests' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('In-Process HTTP Statistics'.center(78)) text_util.print_to_fd('-' * 78) total = int(self.results['total_requests']) numerrors = int(self.results['request_errors']) numbreaks = int(self.results['connection_breaks']) availability = (((total - numerrors) / float(total)) * 100 if total > 0 else 100) text_util.print_to_fd('Total HTTP requests made: %d' % total) text_util.print_to_fd('HTTP 5xx errors: %d' % numerrors) text_util.print_to_fd('HTTP connections broken: %d' % numbreaks) text_util.print_to_fd('Availability: %.7g%%' % availability) if 'error_responses_by_code' in self.results: sorted_codes = sorted( six.iteritems(self.results['error_responses_by_code'])) if sorted_codes: text_util.print_to_fd('Error responses by code:') text_util.print_to_fd('\n'.join(' %s: %s' % c for c in sorted_codes)) if self.output_file: with open(self.output_file, 'w') as f: json.dump(self.results, f, indent=2) text_util.print_to_fd() text_util.print_to_fd("Output file written to '%s'." % self.output_file) text_util.print_to_fd() def _ParsePositiveInteger(self, val, msg): """Tries to convert val argument to a positive integer. Args: val: The value (as a string) to convert to a positive integer. msg: The error message to place in the CommandException on an error. Returns: A valid positive integer. Raises: CommandException: If the supplied value is not a valid positive integer. """ try: val = int(val) if val < 1: raise CommandException(msg) return val except ValueError: raise CommandException(msg) def _ParseArgs(self): """Parses arguments for perfdiag command.""" # From -n. self.num_objects = 5 # From -c. self.processes = 1 # From -k. self.threads = 1 # From -p self.parallel_strategy = None # From -y self.num_slices = 4 # From -s. self.thru_filesize = 1048576 # From -d. self.directory = tempfile.gettempdir() # Keep track of whether or not to delete the directory upon completion. self.delete_directory = False # From -t. self.diag_tests = set(self.DEFAULT_DIAG_TESTS) # From -o. self.output_file = None # From -i. self.input_file = None # From -m. self.metadata_keys = {} # From -j. self.gzip_encoded_writes = False self.gzip_compression_ratio = 100 if self.sub_opts: for o, a in self.sub_opts: if o == '-n': self.num_objects = self._ParsePositiveInteger( a, 'The -n parameter must be a positive integer.') if o == '-c': self.processes = self._ParsePositiveInteger( a, 'The -c parameter must be a positive integer.') if o == '-k': self.threads = self._ParsePositiveInteger( a, 'The -k parameter must be a positive integer.') if o == '-p': if a.lower() in self.PARALLEL_STRATEGIES: self.parallel_strategy = a.lower() else: raise CommandException("'%s' is not a valid parallelism strategy." % a) if o == '-y': self.num_slices = self._ParsePositiveInteger( a, 'The -y parameter must be a positive integer.') if o == '-s': try: self.thru_filesize = HumanReadableToBytes(a) except ValueError: raise CommandException('Invalid -s parameter.') if o == '-d': self.directory = a if not os.path.exists(self.directory): self.delete_directory = True os.makedirs(self.directory) if o == '-t': self.diag_tests = set() for test_name in a.strip().split(','): if test_name.lower() not in self.ALL_DIAG_TESTS: raise CommandException("List of test names (-t) contains invalid " "test name '%s'." % test_name) self.diag_tests.add(test_name) if o == '-m': pieces = a.split(':') if len(pieces) != 2: raise CommandException( "Invalid metadata key-value combination '%s'." % a) key, value = pieces self.metadata_keys[key] = value if o == '-o': self.output_file = os.path.abspath(a) if o == '-i': self.input_file = os.path.abspath(a) if not os.path.isfile(self.input_file): raise CommandException("Invalid input file (-i): '%s'." % a) try: with open(self.input_file, 'r') as f: self.results = json.load(f) self.logger.info("Read input file: '%s'.", self.input_file) except ValueError: raise CommandException("Could not decode input file (-i): '%s'." % a) return if o == '-j': self.gzip_encoded_writes = True try: self.gzip_compression_ratio = int(a) except ValueError: self.gzip_compression_ratio = -1 if (self.gzip_compression_ratio < 0 or self.gzip_compression_ratio > 100): raise CommandException( 'The -j parameter must be between 0 and 100 (inclusive).') # If parallelism is specified, default parallelism strategy to fan. if (self.processes > 1 or self.threads > 1) and not self.parallel_strategy: self.parallel_strategy = self.FAN elif self.processes == 1 and self.threads == 1 and self.parallel_strategy: raise CommandException( 'Cannot specify parallelism strategy (-p) without also specifying ' 'multiple threads and/or processes (-c and/or -k).') if not self.args: self.RaiseWrongNumberOfArgumentsException() self.bucket_url = StorageUrlFromString(self.args[0]) self.provider = self.bucket_url.scheme if not self.bucket_url.IsCloudUrl() and self.bucket_url.IsBucket(): raise CommandException('The perfdiag command requires a URL that ' 'specifies a bucket.\n"%s" is not ' 'valid.' % self.args[0]) if (self.thru_filesize > HumanReadableToBytes('2GiB') and (self.RTHRU in self.diag_tests or self.WTHRU in self.diag_tests)): raise CommandException( 'For in-memory tests maximum file size is 2GiB. For larger file ' 'sizes, specify rthru_file and/or wthru_file with the -t option.') perform_slice = self.parallel_strategy in (self.SLICE, self.BOTH) slice_not_available = (self.provider == 's3' and self.diag_tests.intersection(self.WTHRU, self.WTHRU_FILE)) if perform_slice and slice_not_available: raise CommandException('Sliced uploads are not available for s3. ' 'Use -p fan or sequential uploads for s3.') # Ensure the bucket exists. self.gsutil_api.GetBucket(self.bucket_url.bucket_name, provider=self.bucket_url.scheme, fields=['id']) self.exceptions = [ http_client.HTTPException, socket.error, socket.gaierror, socket.timeout, http_client.BadStatusLine, ServiceException ] # Command entry point. def RunCommand(self): """Called by gsutil when the command is being invoked.""" self._ParseArgs() if self.input_file: self._DisplayResults() return 0 # We turn off retries in the underlying boto library because the # _RunOperation function handles errors manually so it can count them. boto.config.set('Boto', 'num_retries', '0') self.logger.info( 'Number of iterations to run: %d\n' 'Base bucket URI: %s\n' 'Number of processes: %d\n' 'Number of threads: %d\n' 'Parallelism strategy: %s\n' 'Throughput file size: %s\n' 'Diagnostics to run: %s\n' 'Gzip compression ratio: %s\n' 'Gzip transport encoding writes: %s', self.num_objects, self.bucket_url, self.processes, self.threads, self.parallel_strategy, MakeHumanReadable(self.thru_filesize), (', '.join(self.diag_tests)), self.gzip_compression_ratio, self.gzip_encoded_writes) try: self._SetUp() # Collect generic system info. self._CollectSysInfo() # Collect netstat info and disk counters before tests (and again later). netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_start'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_start'] = GetDiskCounters() # Record bucket URL. self.results['bucket_uri'] = str(self.bucket_url) self.results['json_format'] = 'perfdiag' self.results['metadata'] = self.metadata_keys if self.LAT in self.diag_tests: self._RunLatencyTests() if self.RTHRU in self.diag_tests: self._RunReadThruTests() # Run WTHRU_FILE before RTHRU_FILE. If data is created in WTHRU_FILE it # will be used in RTHRU_FILE to save time and bandwidth. if self.WTHRU_FILE in self.diag_tests: self._RunWriteThruTests(use_file=True) if self.RTHRU_FILE in self.diag_tests: self._RunReadThruTests(use_file=True) if self.WTHRU in self.diag_tests: self._RunWriteThruTests() if self.LIST in self.diag_tests: self._RunListTests() # Collect netstat info and disk counters after tests. netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_end'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_end'] = GetDiskCounters() self.results['total_requests'] = self.total_requests self.results['request_errors'] = self.request_errors self.results['error_responses_by_code'] = self.error_responses_by_code self.results['connection_breaks'] = self.connection_breaks self.results['gsutil_version'] = gslib.VERSION self.results['boto_version'] = boto.__version__ self._TearDown() self._DisplayResults() finally: # TODO: Install signal handlers so this is performed in response to a # terminating signal; consider multi-threaded object deletes during # cleanup so it happens quickly. self._TearDown() return 0 def StorageUrlToUploadObjectMetadata(storage_url): if storage_url.IsCloudUrl() and storage_url.IsObject(): upload_target = apitools_messages.Object() upload_target.name = storage_url.object_name upload_target.bucket = storage_url.bucket_name return upload_target else: raise CommandException('Non-cloud URL upload target %s was created in ' 'perfdiag implemenation.' % storage_url) notification.py 0000644 00000103705 15025166704 0007620 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the notification command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import re import time import uuid from datetime import datetime from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PublishPermissionDeniedException from gslib.command import Command from gslib.command import NO_MAX from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.project_id import PopulateProjectId from gslib.pubsub_api import PubsubApi from gslib.storage_url import StorageUrlFromString from gslib.third_party.pubsub_apitools.pubsub_v1_messages import Binding from gslib.utils import copy_helper from gslib.utils import shim_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap # Cloud Pub/Sub commands _LIST_SYNOPSIS = """ gsutil notification list gs://<bucket_name>... """ _DELETE_SYNOPSIS = """ gsutil notification delete (<notificationConfigName>|gs://<bucket_name>)... """ _CREATE_SYNOPSIS = """ gsutil notification create -f (json|none) [-p <prefix>] [-t <topic>] \\ [-m <key>:<value>]... [-e <eventType>]... gs://<bucket_name> """ # Object Change Notification commands _WATCHBUCKET_SYNOPSIS = """ gsutil notification watchbucket [-i <id>] [-t <token>] <app_url> gs://<bucket_name> """ _STOPCHANNEL_SYNOPSIS = """ gsutil notification stopchannel <channel_id> <resource_id> """ _SYNOPSIS = ( _CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _WATCHBUCKET_SYNOPSIS + _STOPCHANNEL_SYNOPSIS.lstrip('\n') + '\n') # yapf: disable _LIST_DESCRIPTION = """ <B>LIST</B> The list sub-command provides a list of notification configs belonging to a given bucket. The listed name of each notification config can be used with the delete sub-command to delete that specific notification config. For listing Object Change Notifications instead of Cloud Pub/Sub notification subscription configs, add a -o flag. <B>LIST EXAMPLES</B> Fetch the list of notification configs for the bucket example-bucket: gsutil notification list gs://example-bucket The same as above, but for Object Change Notifications instead of Cloud Pub/Sub notification subscription configs: gsutil notification list -o gs://example-bucket Fetch the notification configs in all buckets matching a wildcard: gsutil notification list gs://example-* Fetch all of the notification configs for buckets in the default project: gsutil notification list gs://* """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The delete sub-command deletes notification configs from a bucket. If a notification config name is passed as a parameter, that notification config alone is deleted. If a bucket name is passed, all notification configs associated with that bucket are deleted. Cloud Pub/Sub topics associated with this notification config are not deleted by this command. Those must be deleted separately, for example with the gcloud command `gcloud beta pubsub topics delete`. Object Change Notification subscriptions cannot be deleted with this command. For that, see the command `gsutil notification stopchannel`. <B>DELETE EXAMPLES</B> Delete a single notification config (with ID 3) in the bucket example-bucket: gsutil notification delete projects/_/buckets/example-bucket/notificationConfigs/3 Delete all notification configs in the bucket example-bucket: gsutil notification delete gs://example-bucket """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The create sub-command creates a notification config on a bucket, establishing a flow of event notifications from Cloud Storage to a Cloud Pub/Sub topic. As part of creating this flow, the create command also verifies that the destination Cloud Pub/Sub topic exists, creating it if necessary, and verifies that the Cloud Storage bucket has permission to publish events to that topic, granting the permission if necessary. If a destination Cloud Pub/Sub topic is not specified with the -t flag, Cloud Storage chooses a topic name in the default project whose ID is the same as the bucket name. For example, if the default project ID specified is 'default-project' and the bucket being configured is gs://example-bucket, the create command uses the Cloud Pub/Sub topic "projects/default-project/topics/example-bucket". In order to enable notifications, your project's `Cloud Storage service agent <https://cloud.google.com/storage/docs/projects#service-accounts>`_ must have the IAM permission "pubsub.topics.publish". This command checks to see if the destination Cloud Pub/Sub topic grants the service agent this permission. If not, the create command attempts to grant it. A bucket can have up to 100 total notification configurations and up to 10 notification configurations set to trigger for a specific event. <B>CREATE EXAMPLES</B> Begin sending notifications of all changes to the bucket example-bucket to the Cloud Pub/Sub topic projects/default-project/topics/example-bucket: gsutil notification create -f json gs://example-bucket The same as above, but specifies the destination topic ID 'files-to-process' in the default project: gsutil notification create -f json \\ -t files-to-process gs://example-bucket The same as above, but specifies a Cloud Pub/Sub topic belonging to the specific cloud project 'example-project': gsutil notification create -f json \\ -t projects/example-project/topics/files-to-process gs://example-bucket Create a notification config that only sends an event when a new object has been created: gsutil notification create -f json -e OBJECT_FINALIZE gs://example-bucket Create a topic and notification config that only sends an event when an object beginning with "photos/" is affected: gsutil notification create -p photos/ gs://example-bucket List all of the notificationConfigs in bucket example-bucket: gsutil notification list gs://example-bucket Delete all notitificationConfigs for bucket example-bucket: gsutil notification delete gs://example-bucket Delete one specific notificationConfig for bucket example-bucket: gsutil notification delete \\ projects/_/buckets/example-bucket/notificationConfigs/1 <B>OPTIONS</B> The create sub-command has the following options -e Specify an event type filter for this notification config. Cloud Storage only sends notifications of this type. You may specify this parameter multiple times to allow multiple event types. If not specified, Cloud Storage sends notifications for all event types. The valid types are: OBJECT_FINALIZE - An object has been created. OBJECT_METADATA_UPDATE - The metadata of an object has changed. OBJECT_DELETE - An object has been permanently deleted. OBJECT_ARCHIVE - A live version of an object has become a noncurrent version. -f Specifies the payload format of notification messages. Must be either "json" for a payload matches the object metadata for the JSON API, or "none" to specify no payload at all. In either case, notification details are available in the message attributes. -m Specifies a key:value attribute that is appended to the set of attributes sent to Cloud Pub/Sub for all events associated with this notification config. You may specify this parameter multiple times to set multiple attributes. -p Specifies a prefix path filter for this notification config. Cloud Storage only sends notifications for objects in this bucket whose names begin with the specified prefix. -s Skips creation and permission assignment of the Cloud Pub/Sub topic. This is useful if the caller does not have permission to access the topic in question, or if the topic already exists and has the appropriate publish permission assigned. -t The Cloud Pub/Sub topic to which notifications should be sent. If not specified, this command chooses a topic whose project is your default project and whose ID is the same as the Cloud Storage bucket name. <B>NEXT STEPS</B> Once the create command has succeeded, Cloud Storage publishes a message to the specified Cloud Pub/Sub topic when eligible changes occur. In order to receive these messages, you must create a Pub/Sub subscription for your Pub/Sub topic. To learn more about creating Pub/Sub subscriptions, see `the Pub/Sub Subscriber Overview <https://cloud.google.com/pubsub/docs/subscriber>`_. You can create a simple Pub/Sub subscription using the ``gcloud`` command-line tool. For example, to create a new subscription on the topic "myNewTopic" and attempt to pull messages from it, you could run: gcloud beta pubsub subscriptions create --topic myNewTopic testSubscription gcloud beta pubsub subscriptions pull --auto-ack testSubscription """ _WATCHBUCKET_DESCRIPTION = """ <B>WATCHBUCKET</B> The watchbucket sub-command can be used to watch a bucket for object changes. A service account must be used when running this command. The app_url parameter must be an HTTPS URL to an application that will be notified of changes to any object in the bucket. The optional id parameter can be used to assign a unique identifier to the created notification channel. If not provided, a random UUID string is generated. The optional token parameter can be used to validate notifications events. To do this, set this custom token and store it to later verify that notification events contain the client token you expect. <B>WATCHBUCKET EXAMPLES</B> Watch the bucket example-bucket for changes and send notifications to an application server running at example.com: gsutil notification watchbucket https://example.com/notify \\ gs://example-bucket Assign identifier my-channel-id to the created notification channel: gsutil notification watchbucket -i my-channel-id \\ https://example.com/notify gs://example-bucket Set a custom client token that is included with each notification event: gsutil notification watchbucket -t my-client-token \\ https://example.com/notify gs://example-bucket """ _STOPCHANNEL_DESCRIPTION = """ <B>STOPCHANNEL</B> The stopchannel sub-command can be used to stop sending change events to a notification channel. The channel_id and resource_id parameters should match the values from the response of a bucket watch request. <B>STOPCHANNEL EXAMPLES</B> Stop the notification event channel with channel identifier channel1 and resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8: gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8 """ _DESCRIPTION = """ You can use the ``notification`` command to configure `Pub/Sub notifications for Cloud Storage <https://cloud.google.com/storage/docs/pubsub-notifications>`_ and `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_ channels. <B>CLOUD PUB/SUB</B> The "create", "list", and "delete" sub-commands deal with configuring Cloud Storage integration with Google Cloud Pub/Sub. """ + _CREATE_DESCRIPTION + _LIST_DESCRIPTION + _DELETE_DESCRIPTION + """ <B>OBJECT CHANGE NOTIFICATIONS</B> Object change notification is a separate, older feature within Cloud Storage for generating notifications. This feature sends HTTPS messages to a client application that you've set up separately. This feature is generally not recommended, because Pub/Sub notifications are cheaper, easier to use, and more flexible. For more information, see `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_. The "watchbucket" and "stopchannel" sub-commands enable and disable Object change notifications. """ + _WATCHBUCKET_DESCRIPTION + _STOPCHANNEL_DESCRIPTION + """ <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B> gsutil supports `parallel composite uploads <https://cloud.google.com/storage/docs/uploads-downloads#parallel-composite-uploads>`_. If enabled, an upload can result in multiple temporary component objects being uploaded before the actual intended object is created. Any subscriber to notifications for this bucket then sees a notification for each of these components being created and deleted. If this is a concern for you, note that parallel composite uploads can be disabled by setting "parallel_composite_upload_threshold = 0" in your .boto config file. Alternately, your subscriber code can filter out gsutil's parallel composite uploads by ignoring any notification about objects whose names contain (but do not start with) the following string: "{composite_namespace}". """.format(composite_namespace=copy_helper.PARALLEL_UPLOAD_TEMP_NAMESPACE) NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE = """ Watch bucket attempt failed: {watch_error} You attempted to watch a bucket with an application URL of: {watch_url} which is not authorized for your project. Please ensure that you are using Service Account authentication and that the Service Account's project is authorized for the application URL. Notification endpoint URLs must also be whitelisted in your Cloud Console project. To do that, the domain must also be verified using Google Webmaster Tools. For instructions, please see `Notification Authorization <https://cloud.google.com/storage/docs/object-change-notification#_Authorization>`_. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) # yapf: disable _create_help_text = ( CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION)) _list_help_text = ( CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION)) _delete_help_text = ( CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION)) _watchbucket_help_text = ( CreateHelpText(_WATCHBUCKET_SYNOPSIS, _WATCHBUCKET_DESCRIPTION)) _stopchannel_help_text = ( CreateHelpText(_STOPCHANNEL_SYNOPSIS, _STOPCHANNEL_DESCRIPTION)) # yapf: enable PAYLOAD_FORMAT_MAP = { 'none': 'NONE', 'json': 'JSON_API_V1', } class NotificationCommand(Command): """Implementation of gsutil notification command.""" # Notification names might look like one of these: # canonical form: projects/_/buckets/bucket/notificationConfigs/3 # JSON API form: b/bucket/notificationConfigs/5 # Either of the above might start with a / if a user is copying & pasting. def _GetNotificationPathRegex(self): if not NotificationCommand._notification_path_regex: NotificationCommand._notification_path_regex = re.compile( ('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/' 'notificationConfigs/(?P<notification>[0-9]+)')) return NotificationCommand._notification_path_regex _notification_path_regex = None # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'notification', command_name_aliases=[ 'notify', 'notifyconfig', 'notifications', 'notif', ], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='i:t:m:t:of:e:p:s', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'watchbucket': [ CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'stopchannel': [], 'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], 'delete': [ # Takes a list of one of the following: # notification: projects/_/buckets/bla/notificationConfigs/5, # bucket: gs://foobar CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'create': [ CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic CommandArgument.MakeNCloudBucketURLsArgument(1), ] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='notification', help_name_aliases=[ 'watchbucket', 'stopchannel', 'notifyconfig', ], help_type='command_help', help_one_line_summary='Configure object change notification', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'list': _list_help_text, 'delete': _delete_help_text, 'watchbucket': _watchbucket_help_text, 'stopchannel': _stopchannel_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'create' ], flag_map={ '-m': GcloudStorageFlag( '--custom-attributes', repeat_type=shim_util.RepeatFlagType.DICT), '-e': GcloudStorageFlag( '--event-types', repeat_type=shim_util.RepeatFlagType.LIST), '-p': GcloudStorageFlag('--object-prefix'), '-f': GcloudStorageFlag('--payload-format'), '-s': GcloudStorageFlag('--skip-topic-setup'), '-t': GcloudStorageFlag('--topic'), }, ), 'delete': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'delete' ], flag_map={}, ), 'list': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'list', '--human-readable', ], flag_map={}, supports_output_translation=True, ), }, flag_map={}, ) def _WatchBucket(self): """Creates a watch on a bucket given in self.args.""" self.CheckArguments() identifier = None client_token = None if self.sub_opts: for o, a in self.sub_opts: if o == '-i': identifier = a if o == '-t': client_token = a identifier = identifier or str(uuid.uuid4()) watch_url = self.args[0] bucket_arg = self.args[-1] if not watch_url.lower().startswith('https://'): raise CommandException('The application URL must be an https:// URL.') bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) self.logger.info('Watching bucket %s with application URL %s ...', bucket_url, watch_url) try: channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name, watch_url, identifier, token=client_token, provider=bucket_url.scheme) except AccessDeniedException as e: self.logger.warn( NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(watch_error=str(e), watch_url=watch_url)) raise channel_id = channel.id resource_id = channel.resourceId client_token = channel.token self.logger.info('Successfully created watch notification channel.') self.logger.info('Watch channel identifier: %s', channel_id) self.logger.info('Canonicalized resource identifier: %s', resource_id) self.logger.info('Client state token: %s', client_token) return 0 def _StopChannel(self): channel_id = self.args[0] resource_id = self.args[1] self.logger.info('Removing channel %s with resource identifier %s ...', channel_id, resource_id) self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs') self.logger.info('Succesfully removed channel.') return 0 def _ListChannels(self, bucket_arg): """Lists active channel watches on a bucket given in self.args.""" bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) channels = self.gsutil_api.ListChannels(bucket_url.bucket_name, provider='gs').items self.logger.info( 'Bucket %s has the following active Object Change Notifications:', bucket_url.bucket_name) for idx, channel in enumerate(channels): self.logger.info('\tNotification channel %d:', idx + 1) self.logger.info('\t\tChannel identifier: %s', channel.channel_id) self.logger.info('\t\tResource identifier: %s', channel.resource_id) self.logger.info('\t\tApplication URL: %s', channel.push_url) self.logger.info('\t\tCreated by: %s', channel.subscriber_email) self.logger.info( '\t\tCreation time: %s', str(datetime.fromtimestamp(channel.creation_time_ms / 1000))) return 0 def _Create(self): self.CheckArguments() # User-specified options pubsub_topic = None payload_format = None custom_attributes = {} event_types = [] object_name_prefix = None should_setup_topic = True if self.sub_opts: for o, a in self.sub_opts: if o == '-e': event_types.append(a) elif o == '-f': payload_format = a elif o == '-m': if ':' not in a: raise CommandException( 'Custom attributes specified with -m should be of the form ' 'key:value') key, value = a.split(':', 1) custom_attributes[key] = value elif o == '-p': object_name_prefix = a elif o == '-s': should_setup_topic = False elif o == '-t': pubsub_topic = a if payload_format not in PAYLOAD_FORMAT_MAP: raise CommandException( "Must provide a payload format with -f of either 'json' or 'none'") payload_format = PAYLOAD_FORMAT_MAP[payload_format] bucket_arg = self.args[-1] bucket_url = StorageUrlFromString(bucket_arg) if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket(): raise CommandException( "%s %s requires a GCS bucket name, but got '%s'" % (self.command_name, self.subcommand_name, bucket_arg)) if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) bucket_name = bucket_url.bucket_name self.logger.debug('Creating notification for bucket %s', bucket_url) # Find the project this bucket belongs to bucket_metadata = self.gsutil_api.GetBucket(bucket_name, fields=['projectNumber'], provider=bucket_url.scheme) bucket_project_number = bucket_metadata.projectNumber # If not specified, choose a sensible default for the Cloud Pub/Sub topic # name. if not pubsub_topic: pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), bucket_name) if not pubsub_topic.startswith('projects/'): # If a user picks a topic ID (mytopic) but doesn't pass the whole name ( # projects/my-project/topics/mytopic ), pick a default project. pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), pubsub_topic) self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic) just_modified_topic_permissions = False if should_setup_topic: # Ask GCS for the email address that represents GCS's permission to # publish to a Cloud Pub/Sub topic from this project. service_account = self.gsutil_api.GetProjectServiceAccount( bucket_project_number, provider=bucket_url.scheme).email_address self.logger.debug('Service account for project %d: %s', bucket_project_number, service_account) just_modified_topic_permissions = self._CreateTopic( pubsub_topic, service_account) for attempt_number in range(0, 2): try: create_response = self.gsutil_api.CreateNotificationConfig( bucket_name, pubsub_topic=pubsub_topic, payload_format=payload_format, custom_attributes=custom_attributes, event_types=event_types if event_types else None, object_name_prefix=object_name_prefix, provider=bucket_url.scheme) break except PublishPermissionDeniedException: if attempt_number == 0 and just_modified_topic_permissions: # If we have just set the IAM policy, it may take up to 10 seconds to # take effect. self.logger.info( 'Retrying create notification in 10 seconds ' '(new permissions may take up to 10 seconds to take effect.)') time.sleep(10) else: raise notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % ( bucket_name, create_response.id) self.logger.info('Created notification config %s', notification_name) return 0 def _CreateTopic(self, pubsub_topic, service_account): """Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. """ pubsub_api = PubsubApi(logger=self.logger) # Verify that the Pub/Sub topic exists. If it does not, create it. try: pubsub_api.GetTopic(topic_name=pubsub_topic) self.logger.debug('Topic %s already exists', pubsub_topic) except NotFoundException: self.logger.debug('Creating topic %s', pubsub_topic) pubsub_api.CreateTopic(topic_name=pubsub_topic) self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic) # Verify that the service account is in the IAM policy. policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic) binding = Binding(role='roles/pubsub.publisher', members=['serviceAccount:%s' % service_account]) # This could be more extensive. We could, for instance, check for roles # that are stronger that pubsub.publisher, like owner. We could also # recurse up the hierarchy looking to see if there are project-level # permissions. This can get very complex very quickly, as the caller # may not necessarily have access to the project-level IAM policy. # There's no danger in double-granting permission just to make sure it's # there, though. if binding not in policy.bindings: policy.bindings.append(binding) # transactional safety via etag field. pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic, policy=policy) return True else: self.logger.debug('GCS already has publish permission to topic %s.', pubsub_topic) return False def _EnumerateNotificationsFromArgs(self, accept_notification_configs=True): """Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) """ path_regex = self._GetNotificationPathRegex() for list_entry in self.args: match = path_regex.match(list_entry) if match: if not accept_notification_configs: raise CommandException( '%s %s accepts only bucket names, but you provided %s' % (self.command_name, self.subcommand_name, list_entry)) bucket_name = match.group('bucket') notification_id = match.group('notification') found = False for notification in self.gsutil_api.ListNotificationConfigs( bucket_name, provider='gs'): if notification.id == notification_id: yield (bucket_name, notification) found = True break if not found: raise NotFoundException('Could not find notification %s' % list_entry) else: storage_url = StorageUrlFromString(list_entry) if not storage_url.IsCloudUrl(): raise CommandException( 'The %s command must be used on cloud buckets or notification ' 'config names.' % self.command_name) if storage_url.scheme != 'gs': raise CommandException('The %s command only works on gs:// buckets.') path = None if storage_url.IsProvider(): path = 'gs://*' elif storage_url.IsBucket(): path = list_entry if not path: raise CommandException( 'The %s command cannot be used on cloud objects, only buckets' % self.command_name) for blr in self.WildcardIterator(path).IterBuckets( bucket_fields=['id']): for notification in self.gsutil_api.ListNotificationConfigs( blr.storage_url.bucket_name, provider='gs'): yield (blr.storage_url.bucket_name, notification) def _List(self): self.CheckArguments() if self.sub_opts: if '-o' in dict(self.sub_opts): for bucket_name in self.args: self._ListChannels(bucket_name) else: for bucket_name, notification in self._EnumerateNotificationsFromArgs( accept_notification_configs=False): self._PrintNotificationDetails(bucket_name, notification) return 0 def _PrintNotificationDetails(self, bucket, notification): print('projects/_/buckets/{bucket}/notificationConfigs/{notification}\n' '\tCloud Pub/Sub topic: {topic}'.format( bucket=bucket, notification=notification.id, topic=notification.topic[len('//pubsub.googleapis.com/'):])) if notification.custom_attributes: print('\tCustom attributes:') for attr in notification.custom_attributes.additionalProperties: print('\t\t%s: %s' % (attr.key, attr.value)) filters = [] if notification.event_types: filters.append('\t\tEvent Types: %s' % ', '.join(notification.event_types)) if notification.object_name_prefix: filters.append("\t\tObject name prefix: '%s'" % notification.object_name_prefix) if filters: print('\tFilters:') for line in filters: print(line) self.logger.info('') def _Delete(self): for bucket_name, notification in self._EnumerateNotificationsFromArgs(): self._DeleteNotification(bucket_name, notification.id) return 0 def _DeleteNotification(self, bucket_name, notification_id): self.gsutil_api.DeleteNotificationConfig(bucket_name, notification=notification_id, provider='gs') return 0 def _RunSubCommand(self, func): try: (self.sub_opts, self.args) = getopt.getopt(self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() SUBCOMMANDS = { 'create': _Create, 'list': _List, 'delete': _Delete, 'watchbucket': _WatchBucket, 'stopchannel': _StopChannel } def RunCommand(self): """Command entry point for the notification command.""" self.subcommand_name = self.args.pop(0) if self.subcommand_name in NotificationCommand.SUBCOMMANDS: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand( NotificationCommand.SUBCOMMANDS[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name)) test.py 0000644 00000070440 15025166704 0006110 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil test command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple import logging import os import subprocess import re import sys import tempfile import textwrap import time import traceback import six from six.moves import range import gslib from gslib.cloud_api import ProjectIdException from gslib.command import Command from gslib.command import ResetFailureCount from gslib.exception import CommandException from gslib.project_id import PopulateProjectId import gslib.tests as tests from gslib.tests.util import GetTestNames from gslib.tests.util import InvokedFromParFile from gslib.tests.util import unittest from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS # pylint: disable=g-import-not-at-top try: import coverage except ImportError: coverage = None if six.PY3: long = int _DEFAULT_TEST_PARALLEL_PROCESSES = 5 _DEFAULT_S3_TEST_PARALLEL_PROCESSES = 50 _SEQUENTIAL_ISOLATION_FLAG = 'sequential_only' _SYNOPSIS = """ gsutil test [-l] [-u] [-f] [command command...] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil test command runs the gsutil unit tests and integration tests. The unit tests use an in-memory mock storage service implementation, while the integration tests send requests to the production service using the `preferred API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ set in the boto configuration file. CAUTION: The ``test`` command creates test buckets and objects in your project. Force quitting the ``test`` command can leave behind stale buckets, objects, and HMAC keys in your project. To run both the unit tests and integration tests, run the command with no arguments: gsutil test To run the unit tests only (which run quickly): gsutil test -u Tests run in parallel regardless of whether the top-level -m flag is present. To limit the number of tests run in parallel to 10 at a time: gsutil test -p 10 To force tests to run sequentially: gsutil test -p 1 To have sequentially-run tests stop running immediately when an error occurs: gsutil test -f To run tests for one or more individual commands add those commands as arguments. For example, the following command will run the cp and mv command tests: gsutil test cp mv To list available tests, run the test command with the -l argument: gsutil test -l The tests are defined in the code under the gslib/tests module. Each test file is of the format test_[name].py where [name] is the test name you can pass to this command. For example, running "gsutil test ls" would run the tests in "gslib/tests/test_ls.py". You can also run an individual test class or function name by passing the test module followed by the class name and optionally a test name. For example, to run the an entire test class by name: gsutil test naming.GsutilNamingTests or an individual test function: gsutil test cp.TestCp.test_streaming You can list the available tests under a module or class by passing arguments with the -l option. For example, to list all available test functions in the cp module: gsutil test -l cp To output test coverage: gsutil test -c -p 500 coverage html This will output an HTML report to a directory named 'htmlcov'. Test coverage is compatible with v4.1 of the coverage module (https://pypi.python.org/pypi/coverage). <B>OPTIONS</B> -b Run tests against multi-regional US buckets. By default, tests run against regional buckets in us-central1. -c Output coverage information. -f Exit on first sequential test failure. -l List available tests. -p N Run at most N tests in parallel. The default value is %d. -s Run tests against S3 instead of GS. -u Only run unit tests. """ % _DEFAULT_TEST_PARALLEL_PROCESSES) TestProcessData = namedtuple('TestProcessData', 'name return_code stdout stderr') def MakeCustomTestResultClass(total_tests): """Creates a closure of CustomTestResult. Args: total_tests: The total number of tests being run. Returns: An instance of CustomTestResult. """ class CustomTestResult(unittest.TextTestResult): """A subclass of unittest.TextTestResult that prints a progress report.""" def startTest(self, test): super(CustomTestResult, self).startTest(test) if self.dots: test_id = '.'.join(test.id().split('.')[-2:]) message = ('\r%d/%d finished - E[%d] F[%d] s[%d] - %s' % (self.testsRun, total_tests, len(self.errors), len(self.failures), len(self.skipped), test_id)) message = message[:73] message = message.ljust(73) self.stream.write('%s - ' % message) return CustomTestResult def GetTestNamesFromSuites(test_suite): """Takes a list of test suites and returns a list of contained test names.""" suites = [test_suite] test_names = [] while suites: suite = suites.pop() for test in suite: if isinstance(test, unittest.TestSuite): suites.append(test) else: test_names.append(test.id()[len('gslib.tests.test_'):]) return test_names # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def TestCaseToName(test_case): """Converts a python.unittest to its gsutil test-callable name.""" return (str(test_case.__class__).split('\'')[1] + '.' + test_case._testMethodName) # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def SplitParallelizableTestSuite(test_suite): """Splits a test suite into groups with different running properties. Args: test_suite: A python unittest test suite. Returns: 4-part tuple of lists of test names: (tests that must be run sequentially, tests that must be isolated in a separate process but can be run either sequentially or in parallel, unit tests that can be run in parallel, integration tests that can run in parallel) """ # pylint: disable=import-not-at-top # Need to import this after test globals are set so that skip functions work. from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase isolated_tests = [] sequential_tests = [] parallelizable_integration_tests = [] parallelizable_unit_tests = [] items_to_evaluate = [test_suite] cases_to_evaluate = [] # Expand the test suites into individual test cases: while items_to_evaluate: suite_or_case = items_to_evaluate.pop() if isinstance(suite_or_case, unittest.suite.TestSuite): for item in suite_or_case._tests: items_to_evaluate.append(item) elif isinstance(suite_or_case, unittest.TestCase): cases_to_evaluate.append(suite_or_case) for test_case in cases_to_evaluate: test_method = getattr(test_case, test_case._testMethodName, None) if getattr(test_method, 'requires_isolation', False): # Test must be isolated to a separate process, even it if is being # run sequentially. isolated_tests.append(TestCaseToName(test_case)) elif not getattr(test_method, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif not getattr(test_case, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif isinstance(test_case, GsUtilUnitTestCase): parallelizable_unit_tests.append(TestCaseToName(test_case)) else: parallelizable_integration_tests.append(TestCaseToName(test_case)) return (sorted(sequential_tests), sorted(isolated_tests), sorted(parallelizable_unit_tests), sorted(parallelizable_integration_tests)) def CountFalseInList(input_list): """Counts number of falses in the input list.""" num_false = 0 for item in input_list: if not item: num_false += 1 return num_false def CreateTestProcesses(parallel_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=None): """Creates test processes to run tests in parallel. Args: parallel_tests: List of all parallel tests. test_index: List index of last created test before this function call. process_list: List of running subprocesses. Created processes are appended to this list. process_done: List of booleans indicating process completion. One 'False' will be added per process created. max_parallel_tests: Maximum number of tests to run in parallel. root_coverage_file: The root .coverage filename if coverage is requested. Returns: Index of last created test. """ orig_test_index = test_index # checking to see if test was invoked from a par file (bundled archive) # if not, add python executable path to ensure correct version of python # is used for testing executable_prefix = [sys.executable] if not InvokedFromParFile() else [] s3_argument = ['-s'] if tests.util.RUN_S3_TESTS else [] multiregional_buckets = ['-b'] if tests.util.USE_MULTIREGIONAL_BUCKETS else [] project_id_arg = [] try: project_id_arg = [ '-o', 'GSUtil:default_project_id=%s' % PopulateProjectId() ] except ProjectIdException: # If we don't have a project ID, unit tests should still be able to pass. pass process_create_start_time = time.time() last_log_time = process_create_start_time while (CountFalseInList(process_done) < max_parallel_tests and test_index < len(parallel_tests)): env = os.environ.copy() if root_coverage_file: env['GSUTIL_COVERAGE_OUTPUT_FILE'] = root_coverage_file envstr = dict() # constructing command list and ensuring each part is str cmd = [ six.ensure_str(part) for part in list( executable_prefix + [gslib.GSUTIL_PATH] + project_id_arg + ['test'] + s3_argument + multiregional_buckets + ['--' + _SEQUENTIAL_ISOLATION_FLAG] + [parallel_tests[test_index][len('gslib.tests.test_'):]] ) ] # yapf: disable for k, v in six.iteritems(env): envstr[six.ensure_str(k)] = six.ensure_str(v) process_list.append( subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=envstr)) test_index += 1 process_done.append(False) if time.time() - last_log_time > 5: print(('Created %d new processes (total %d/%d created)' % (test_index - orig_test_index, len(process_list), len(parallel_tests)))) last_log_time = time.time() if test_index == len(parallel_tests): print(('Test process creation finished (%d/%d created)' % (len(process_list), len(parallel_tests)))) return test_index class TestCommand(Command): """Implementation of gsutil test command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'test', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='buflp:sc', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, supported_private_args=[_SEQUENTIAL_ISOLATION_FLAG], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='test', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Run gsutil unit/integration tests (for developers)'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunParallelTests(self, parallel_integration_tests, max_parallel_tests, coverage_filename): """Executes the parallel/isolated portion of the test suite. Args: parallel_integration_tests: List of tests to execute. max_parallel_tests: Maximum number of parallel tests to run at once. coverage_filename: If not None, filename for coverage output. Returns: (int number of test failures, float elapsed time) """ process_list = [] process_done = [] process_results = [] # Tuples of (name, return code, stdout, stderr) num_parallel_failures = 0 # Number of logging cycles we ran with no progress. progress_less_logging_cycles = 0 completed_as_of_last_log = 0 num_parallel_tests = len(parallel_integration_tests) parallel_start_time = last_log_time = time.time() test_index = CreateTestProcesses(parallel_integration_tests, 0, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) while len(process_results) < num_parallel_tests: for proc_num in range(len(process_list)): if process_done[proc_num] or process_list[proc_num].poll() is None: continue process_done[proc_num] = True stdout, stderr = process_list[proc_num].communicate() process_list[proc_num].stdout.close() process_list[proc_num].stderr.close() # TODO: Differentiate test failures from errors. if process_list[proc_num].returncode != 0: num_parallel_failures += 1 process_results.append( TestProcessData(name=parallel_integration_tests[proc_num], return_code=process_list[proc_num].returncode, stdout=stdout, stderr=stderr)) if len(process_list) < num_parallel_tests: test_index = CreateTestProcesses(parallel_integration_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) if len(process_results) < num_parallel_tests: if time.time() - last_log_time > 5: print( '%d/%d finished - %d failures' % (len(process_results), num_parallel_tests, num_parallel_failures)) if len(process_results) == completed_as_of_last_log: progress_less_logging_cycles += 1 else: completed_as_of_last_log = len(process_results) # A process completed, so we made progress. progress_less_logging_cycles = 0 if progress_less_logging_cycles > 4: # Ran 5 or more logging cycles with no progress, let the user # know which tests are running slowly or hanging. still_running = [] for proc_num in range(len(process_list)): if not process_done[proc_num]: still_running.append(parallel_integration_tests[proc_num]) elapsed = time.time() - parallel_start_time print(('{sec} seconds elapsed since beginning parallel tests.\n' 'Still running: {procs}').format( sec=str(int(elapsed)), procs=still_running, )) # TODO: Terminate still-running processes if they # hang for a long time. last_log_time = time.time() time.sleep(1) process_run_finish_time = time.time() if num_parallel_failures: for result in process_results: if result.return_code != 0: new_stderr = result.stderr.split(b'\n') print('Results for failed test %s:' % result.name) for line in new_stderr: print(line.decode(UTF8).strip()) return (num_parallel_failures, (process_run_finish_time - parallel_start_time)) def PrintTestResults(self, num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed): """Prints test results for parallel and sequential tests.""" # TODO: Properly track test skips. print('Parallel tests complete. Success: %s Fail: %s' % (num_parallel_tests - num_parallel_failures, num_parallel_failures)) print(( 'Ran %d tests in %.3fs (%d sequential in %.3fs, %d parallel in %.3fs)' % (num_parallel_tests + num_sequential_tests, float(sequential_time_elapsed + parallel_time_elapsed), num_sequential_tests, float(sequential_time_elapsed), num_parallel_tests, float(parallel_time_elapsed)))) self.PrintSkippedTests(sequential_skipped) print() if not num_parallel_failures and sequential_success: print('OK') else: if num_parallel_failures: print('FAILED (parallel tests)') if not sequential_success: print('FAILED (sequential tests)') # TODO: Parallel skipped tests are never gathered anywhere, this needs implementation in RunParallelTests def PrintSkippedTests(self, sequential_skipped=set(), parallel_skipped=set()): """Prints all skipped tests, and the reasons they were skipped. Takes the union of sequentual_skipped and parallel_skipped, and pretty-prints the resulting methods and reasons. Note that these two arguments are lists of tuples from TestResult.skipped as described here: https://docs.python.org/2/library/unittest.html#unittest.TestResult.skipped Args: sequentual_skipped: An instance of TestResult.skipped. parallel_skipped: An instance of TestResult.skipped. """ if len(sequential_skipped) > 0 or len(parallel_skipped) > 0: sequential_skipped = set(sequential_skipped) parallel_skipped = set(parallel_skipped) all_skipped = sequential_skipped.union(parallel_skipped) print('Tests skipped:') for method, reason in all_skipped: print(' ' + method.id()) print(' Reason: ' + reason) def RunCommand(self): """Command entry point for the test command.""" failfast = False list_tests = False max_parallel_tests = _DEFAULT_TEST_PARALLEL_PROCESSES perform_coverage = False sequential_only = False if self.sub_opts: for o, a in self.sub_opts: if o == '-b': tests.util.USE_MULTIREGIONAL_BUCKETS = True elif o == '-c': perform_coverage = True elif o == '-f': failfast = True elif o == '-l': list_tests = True elif o == ('--' + _SEQUENTIAL_ISOLATION_FLAG): # Called to isolate a single test in a separate process. # Don't try to isolate it again (would lead to an infinite loop). sequential_only = True elif o == '-p': max_parallel_tests = long(a) elif o == '-s': if not tests.util.HAS_S3_CREDS: raise CommandException('S3 tests require S3 credentials. Please ' 'add appropriate credentials to your .boto ' 'file and re-run.') tests.util.RUN_S3_TESTS = True elif o == '-u': tests.util.RUN_INTEGRATION_TESTS = False if perform_coverage and not coverage: raise CommandException( 'Coverage has been requested but the coverage module was not found. ' 'You can install it with "pip install coverage".') if (tests.util.RUN_S3_TESTS and max_parallel_tests > _DEFAULT_S3_TEST_PARALLEL_PROCESSES): self.logger.warn( 'Reducing parallel tests to %d due to S3 maximum bucket ' 'limitations.', _DEFAULT_S3_TEST_PARALLEL_PROCESSES) max_parallel_tests = _DEFAULT_S3_TEST_PARALLEL_PROCESSES test_names = sorted(GetTestNames()) if list_tests and not self.args: print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 # Set list of commands to test if supplied. if self.args: commands_to_test = [] for name in self.args: if name in test_names or name.split('.')[0] in test_names: commands_to_test.append('gslib.tests.test_%s' % name) else: commands_to_test.append(name) else: commands_to_test = ['gslib.tests.test_%s' % name for name in test_names] # Installs a ctrl-c handler that tries to cleanly tear down tests. unittest.installHandler() loader = unittest.TestLoader() if commands_to_test: suite = unittest.TestSuite() for command_name in commands_to_test: try: suite_for_current_command = loader.loadTestsFromName(command_name) suite.addTests(suite_for_current_command) except (ImportError, AttributeError) as e: msg = ('Failed to import test code from file %s. TestLoader provided ' 'this error:\n\n%s' % (command_name, str(e))) # Try to give a better error message; by default, unittest swallows # ImportErrors and only shows that an import failed, not why. E.g.: # "'module' object has no attribute 'test_cp' try: __import__(command_name) except Exception as e: stack_trace = traceback.format_exc() err = re.sub('\\n', '\n ', stack_trace) msg += '\n\nAdditional traceback:\n\n%s' % (err) raise CommandException(msg) if list_tests: test_names = GetTestNamesFromSuites(suite) print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 if logging.getLogger().getEffectiveLevel() <= logging.INFO: verbosity = 1 else: verbosity = 2 logging.disable(logging.ERROR) if perform_coverage: # We want to run coverage over the gslib module, but filter out the test # modules and any third-party code. We also filter out anything under the # temporary directory. Otherwise, the gsutil update test (which copies # code to the temporary directory) gets included in the output. coverage_controller = coverage.coverage(source=['gslib'], omit=[ 'gslib/third_party/*', 'gslib/tests/*', tempfile.gettempdir() + '*', ]) coverage_controller.erase() coverage_controller.start() num_parallel_failures = 0 sequential_success = False (sequential_tests, isolated_tests, parallel_unit_tests, parallel_integration_tests) = (SplitParallelizableTestSuite(suite)) # Since parallel integration tests are run in a separate process, they # won't get the override to tests.util, so skip them here. if not tests.util.RUN_INTEGRATION_TESTS: parallel_integration_tests = [] logging.debug('Sequential tests to run: %s', sequential_tests) logging.debug('Isolated tests to run: %s', isolated_tests) logging.debug('Parallel unit tests to run: %s', parallel_unit_tests) logging.debug('Parallel integration tests to run: %s', parallel_integration_tests) # If we're running an already-isolated test (spawned in isolation by a # previous test process), or we have no parallel tests to run, # just run sequentially. For now, unit tests are always run sequentially. run_tests_sequentially = (sequential_only or (len(parallel_integration_tests) <= 1 and not isolated_tests)) # Disable analytics for the duration of testing. This is set as an # environment variable so that the subprocesses will also not report. os.environ['GSUTIL_TEST_ANALYTICS'] = '1' if run_tests_sequentially: total_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(total_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() else: if max_parallel_tests == 1: # We can't take advantage of parallelism, though we may have tests that # need isolation. sequential_tests += parallel_integration_tests parallel_integration_tests = [] sequential_start_time = time.time() # TODO: For now, run unit tests sequentially because they are fast. # We could potentially shave off several seconds of execution time # by executing them in parallel with the integration tests. if len(sequential_tests) + len(parallel_unit_tests): print('Running %d tests sequentially.' % (len(sequential_tests) + len(parallel_unit_tests))) sequential_tests_to_run = sequential_tests + parallel_unit_tests suite = loader.loadTestsFromNames( sorted([test_name for test_name in sequential_tests_to_run])) num_sequential_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(num_sequential_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() sequential_skipped = ret.skipped else: num_sequential_tests = 0 sequential_success = True sequential_time_elapsed = time.time() - sequential_start_time # At this point, all tests get their own process so just treat the # isolated tests as parallel tests. parallel_integration_tests += isolated_tests num_parallel_tests = len(parallel_integration_tests) if not num_parallel_tests: pass else: sequential_skipped = [] num_processes = min(max_parallel_tests, num_parallel_tests) if num_parallel_tests > 1 and max_parallel_tests > 1: message = 'Running %d tests in parallel mode (%d processes).' if num_processes > _DEFAULT_TEST_PARALLEL_PROCESSES: message += ( ' Please be patient while your CPU is incinerated. ' 'If your machine becomes unresponsive, consider reducing ' 'the amount of parallel test processes by running ' '\'gsutil test -p <num_processes>\'.') print(('\n'.join( textwrap.wrap(message % (num_parallel_tests, num_processes))))) else: print(('Running %d tests sequentially in isolated processes.' % num_parallel_tests)) (num_parallel_failures, parallel_time_elapsed) = self.RunParallelTests( parallel_integration_tests, max_parallel_tests, coverage_controller.data_files.filename if perform_coverage else None) self.PrintTestResults(num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed) if perform_coverage: coverage_controller.stop() coverage_controller.combine() coverage_controller.save() print(('Coverage information was saved to: %s' % coverage_controller.data_files.filename)) # Re-enable analytics to report the test command. os.environ['GSUTIL_TEST_ANALYTICS'] = '0' if sequential_success and not num_parallel_failures: ResetFailureCount() return 0 return 1 __pycache__/alias.cpython-39.opt-1.pyc 0000644 00000012436 15025176534 0013453 0 ustar 00 a �!c� � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dl Zd dlm Z d dl Zd dl Zd dl mZ e�d�ZG dd � d e j�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�commands)�_�dnfc @ sl e Zd ZdZed�Zedd� �Zdd� Zdd� Z d d � Z d d � Z d d� Z dd� Z dd� Zdd� Zdd� ZdS )� AliasCommand)�aliasz List or create command aliasesc C sj | � � }|jdddtd�d� |jdddtd�d� | jdd d g d �td �d � | jdddtd�d� d S )Nz--enable-resolvingF� store_truezenable aliases resolving)�default�action�helpz--disable-resolvingzdisable aliases resolving� subcommand�?�list)�addr �deletezaction to do with aliases)�nargsr �choicesr r �*zcommand[=result]zalias definition)r �metavarr )� add_mutually_exclusive_group� add_argumentr )�parserZ enable_group� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/alias.py� set_argparser* s �� � �zAliasCommand.set_argparserc C sH | j j}| jjdv rd|_tj j�� | _| j� � | jj | _ | � � d S )N)r r T) �cli�demands�optsr Z root_userr �aliasesZAliases� aliases_baseZ _load_aliases�enabledZresolving_enabled�_update_config_from_options)�selfr r r r � configure9 s zAliasCommand.configurec C s� d }| j jr d}t�td�� | j jr8d}t�td�� |d ur�tj�t j j j �sft t j j j d��� t jj�t j j j dd d|i� | j�� s�|| j_d S )NTzAliases are now enabledFzAliases are now disabled�w�mainr" )r Zenable_resolving�logger�infor Zdisable_resolving�os�path�existsr r r ZALIASES_CONF_PATH�open�close�confZ BaseConfigZwrite_raw_configfiler! Z_disabled_by_environr" )r$ r" r r r r# B s � z(AliasCommand._update_config_from_optionsc C s� i }| j jD ]�}|�dd�}|d �� }t|�� �dkrJt�td�|� q |�d�rft�td�|� q t|�dkr�t�td�|� q |d �� ||< q |S )N�=� r zInvalid alias key: %s�-z Alias argument has no value: %s) r r �split�strip�lenr( �warningr � startswith)r$ Z new_aliasesr �cmdr r r �_parse_option_aliasT s z AliasCommand._parse_option_aliasc C s~ t j�tjjj�s&ttjjjd��� ztjj� tjjj�}W n: tj j yx } z t � td�|� W Y d }~d S d }~0 0 |S )Nr& zConfig error: %s)r* r+ r, r r r �ALIASES_USER_PATHr- r. Z AliasesConfig� exceptionsZ ConfigErrorr( r6 r )r$ r/ �er r r �_load_user_aliasese s �z AliasCommand._load_user_aliasesc C s` t tjjjd�}d}|d�|�7 }|d7 }|�� D ] \}}|d�|d�|��7 }q2|�|� d S )Nr& z[main] zenabled = {} z [aliases] z{} = {} � ) r- r r r r: �format�items�join�write)r$ � user_aliasesr" Zfileobj�output�key�valuer r r �_store_user_aliasesp s z AliasCommand._store_user_aliasesc C sP | � � }|j}|d u rd S |�|� | �||j� t�td�d�|� � �� d S )NzAliases added: %s�, ) r= r �updaterG r" r( r) r rA �keys)r$ r r/ rC r r r � add_aliasesy s zAliasCommand.add_aliasesc C s� | � � }|j}|d u rd S g }|D ]<}z||= |�|� W q" ty\ t�td�|� Y q"0 q"| �||j� t�td�d� |�� d S )NzAlias not found: %szAliases deleted: %srH ) r= r �append�KeyErrorr( r) r rG r" rA )r$ �cmdsr/ rC Z valid_cmdsr8 r r r �remove_aliases� s zAliasCommand.remove_aliasesc C s� |g}z| j �|�}W nJ tjjy` } z.t�td�||d�| j j | �� W Y d }~n$d }~0 0 t td�|d�|�f � d S )Nz%s, alias %s="%s"r> z Alias %s='%s') r! �_resolver r; �Errorr( �errorr rA r �print)r$ r8 �argsr< r r r � list_alias� s �zAliasCommand.list_aliasc C s� | j jst�td�� | jjdkrL| �� }|s>tj � td���| � |� d S | jjdkr�| jj }|g krxtj � td���| � |� d S | jj s�| j js�t�td�� d S | j jD ]}| �|� q�n8| jj D ].}|| j jvr�t�td�| � q�| �|� q�d S )Nz Aliases resolving is disabled.r zNo aliases specified.r zNo alias specified.zNo aliases defined.zNo match for alias: %s)r! r" r( r6 r r r r9 r r; rQ rK r rO r r) rU )r$ r rN r8 r r r �run� s2 zAliasCommand.runN)�__name__� __module__� __qualname__r r Zsummary� staticmethodr r% r# r9 r= rG rK rO rU rV r r r r r & s r )Z __future__r r r ZloggingZos.pathr* Zdnf.clir Zdnf.cli.aliasesr Zdnf.confZdnf.exceptionsZdnf.i18nr Z getLoggerr( ZCommandr r r r r �<module> s __pycache__/upgrade.cpython-39.opt-1.pyc 0000644 00000007174 15025176534 0014014 0 ustar 00 a �!c~ � @ st d dl mZ d dl mZ d dlZd dlZd dlZd dlmZ d dl m Z d dl m Z e� d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literalsN)�commands)� OptionParser)�_�dnfc @ sX e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dd� ZdS )�UpgradeCommandzTA class containing methods needed by the cli to execute the update command. )�upgrade�updatez upgrade-toz update-toZ localupdateZupz,upgrade a package or packages on your systemc C s" | j ddtd�tjtd�d� d S )NZpackages�*zPackage to upgradeZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/upgrade.py� set_argparser* s �z UpgradeCommand.set_argparserc C sZ | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� d| _ d| _ d| _dS )z�Verify that conditions are met so that this command can run. These include that there are enabled repositories with gpg keys, and that this command is being run by the root user. TN)�cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo�upgrade_minimal� all_security�skipped_grp_specs)�selfr r r r � configure0 s zUpgradeCommand.configurec C s� | j r dnd}| jj| j|| jd� | jjs<| jjs<| jjrzd}|| �� O }|| � � O }|| � � O }|| � � O }|r�d S n| j � � d S tj�td���d S )N�eqZgte)�cmp_type�allFz No packages marked for upgrade.)r r Z _populate_update_security_filterr r r � pkg_specs� grp_specs�_update_modules� _update_files�_update_packages�_update_groupsr Z upgrade_allr � exceptions�Errorr )r r �resultr r r �runB s � zUpgradeCommand.runc C sN t | jj�}tjjr6tjj�| j�}|� | jj�| _ n | jj| _ t | j �|kS )N) �lenr r# r r Z WITH_MODULES�module� module_baseZ ModuleBaser r )r Zgroup_specs_numr. r r r r$ V s z UpgradeCommand._update_modulesc C s� d}| j jr�| jj| j jd| jjjd�D ]`}z| j�|� d}W q( tjj y� } z*t � t d�| jjj �|j�� W Y d }~q(d }~0 0 q(|S )NF)�strict�progressT�No match for argument: %s)r r r Zadd_remote_rpms�outputr0 Zpackage_upgrader r( � MarkingError�logger�infor �term�bold�location)r �success�pkg�er r r r% ` s � � z UpgradeCommand._update_filesc C sp d}| j jD ]^}z| j�|� d}W q tjjyh } z(t�t d�| jj j � |�� W Y d }~q d }~0 0 q |S )NFTr1 ) r r" r r r r( r3 r4 r5 r r2 r6 r7 )r r9 Zpkg_specr; r r r r& m s � z UpgradeCommand._update_packagesc C s | j r| j�| j � dS dS )NTF)r r Zenv_group_upgrade)r r r r r' x s z UpgradeCommand._update_groupsN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r+ r$ r% r& r' r r r r r # s r )Z __future__r r ZloggingZdnf.exceptionsr Zdnf.baseZdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Z getLoggerr4 ZCommandr r r r r �<module> s __pycache__/deplist.cpython-39.opt-1.pyc 0000644 00000001643 15025176534 0014024 0 ustar 00 a �!c� � @ sP d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ G dd� de�ZdS ) � )�print_function)�absolute_import)�unicode_literals)�_)�RepoQueryCommandc @ s$ e Zd ZdZdZed�Zdd� ZdS )�DeplistCommandz< The command is alias for 'dnf repoquery --deplist' )�deplistz`[deprecated, use repoquery --deplist] List package's dependencies and what packages provide themc C s t �| � d| j_d S )NT)r � configureZoptsr )�self� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/deplist.pyr " s zDeplistCommand.configureN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr r r r r r s r N) Z __future__r r r Zdnf.i18nr Zdnf.cli.commands.repoqueryr r r r r r �<module> s __pycache__/autoremove.cpython-39.opt-1.pyc 0000644 00000003664 15025176534 0014553 0 ustar 00 a �!c� � @ st d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z d dl Z e � d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_N�dnfc @ sR e Zd Zejejejd�Zdee� � � Z e d�Z e dd� �Zdd� Zdd � Zd S ) �AutoremoveCommand)z autoremove-nz autoremove-nazautoremove-nevra)� autoremovezKremove all unneeded packages that were originally installed as dependenciesc C s" | j ddtd�tjtd�d� d S )NZpackages�*zPackage to removeZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/autoremove.py� set_argparser, s �z AutoremoveCommand.set_argparserc C s\ | j j}d|_d|_d|_t| jj| jj| jj g�rLd| j j _ d|_ d|_n d|_d|_d S )NTF)Zcli�demandsZ resolvingZ root_userZsack_activation�any�opts� grp_specs� pkg_specs� filenames�baseZconfZ clean_requirements_on_removeZ allow_erasingZavailable_reposZfresh_metadata)�selfr r r r � configure2 s zAutoremoveCommand.configurec C sj t | jj| jj| jjg�r\g }| jj| jv r<| j| jj g}| j�|| jj| jj| jj� n | j�� d S )N) r r r r r Zcommand� nevra_formsr r )r Zformsr r r �runB s �zAutoremoveCommand.runN)�__name__� __module__� __qualname__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr �tuple�keys�aliasesr Zsummary� staticmethodr r r r r r r r " s � r )Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.exceptionsr r" ZloggingZ getLogger�loggerZCommandr r r r r �<module> s __pycache__/swap.cpython-39.opt-1.pyc 0000644 00000003633 15025176534 0013333 0 ustar 00 a �!cA � @ s` d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dl Z e � d�Z G dd� dej �Z dS ) � )�absolute_import)�unicode_literals)�_)�commandsN�dnfc @ sL e Zd ZdZdZed�jejj d�Z e dd� �Z dd� Z d d � Zd d � Zd S )� SwapCommandzNA class containing methods needed by the cli to execute the swap command. )Zswapz=run an interactive {prog} mod for remove and install one spec)�progc C s, | j ddtd�d� | j ddtd�d� d S )N� remove_spec�storez The specs that will be removed)�action�help� install_specz The specs that will be installed)� add_argumentr )�parser� r �9/usr/lib/python3.9/site-packages/dnf/cli/commands/swap.py� set_argparser& s �zSwapCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � t� | j| j j g� d S )NT) �cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�baseZ_checkEnabledRepo�optsr )�selfr r r r � configure, s zSwapCommand.configurec C s@ | j j�|�}|d ur<|| j �}| j j�|||g� |�� d S )N)r Z cli_commands�getZ optparserZparse_command_args�run)r Zcmd_str�specZcmd_cls�cmdr r r �_perform5 s zSwapCommand._performc C s$ | � d| jj� | � d| jj� d S )N�install�remove)r r r r )r r r r r < s zSwapCommand.runN)�__name__� __module__� __qualname__�__doc__�aliasesr �formatr �utilZMAIN_PROG_UPPERZsummary� staticmethodr r r r r r r r r s � r )Z __future__r r Zdnf.i18nr Zdnf.clir Zdnf.utilr ZloggingZ getLogger�loggerZCommandr r r r r �<module> s __pycache__/reinstall.cpython-39.opt-1.pyc 0000644 00000005702 15025176534 0014355 0 ustar 00 a �!c] � @ sl d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z e � d�Z G dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_N�dnfc @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �ReinstallCommandzSA class containing methods needed by the cli to execute the reinstall command. )� reinstallZreizreinstall a packagec C s" | j ddtd�tjtd�d� d S )N�packages�+zPackage to reinstallZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/reinstall.py� set_argparser( s �z ReinstallCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� dS )a Verify that conditions are met so that this command can run. These include that the program is being run by the root user, that there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. TN) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure. s zReinstallCommand.configurec C s� d}| j j| jjd| j jjd�D ]N}z| j �|� W n4 tjj yh t � t d�| j jj �|j�� Y q 0 d}q | jjdd� | jjD � D �] }z| j �|� W �n tjj�y } zT|jD ]$}t � t d�| jj �|j�� q�q�t � t d�| j jj �|�� W Y d }~q�d }~0 tjj�y� } z`|jD ]L}d}| j j�|�}|�rVt d �| }t d �}t � || j jj �|�|� �q.W Y d }~q�d }~0 tjj �y� Y q�0 d}q�|�s�tj�t d ���d S ) NF)�strict�progresszNo match for argument: %sTc S s g | ] }d | �qS )�@r )�.0�xr r r � <listcomp>L � z(ReinstallCommand.run.<locals>.<listcomp>z(Package %s available, but not installed.� z (from %s)z%Installed package %s%s not available.z!No packages marked for reinstall.)r Zadd_remote_rpmsr r �outputr Zpackage_reinstallr � exceptionsZ MarkingError�logger�infor ZtermZbold�locationZ pkg_specsZ grp_specsr ZPackagesNotInstalledErrorr �nameZPackagesNotAvailableError�historyZrepo�Error)r Zdone�pkgZpkg_spec�errZxmsgZpkgrepo�msgr r r �run= sL � � � � � zReinstallCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r. r r r r r ! s r )Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.exceptionsr ZloggingZ getLoggerr% ZCommandr r r r r �<module> s __pycache__/group.cpython-39.pyc 0000644 00000024211 15025176534 0012551 0 ustar 00 a �!cq: � @ s� d dl mZ d dl mZ d dlmZ d dlmZ d dlmZm Z d dl Z d dlZ d dl Z d dlZ d dlZe�d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)� CompsQuery)�commands)�_�ucdN�dnfc s� e Zd ZdZddddddd�Zdee�� � Zed�Z d dd �Z d Z d Z d d� Z � fdd�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zed#d$� �Zd%d&� Zd'd(� Zd)d*� Z � Z S )+� GroupCommandz; Single sub-command interface for most groups interaction. �list�install�remove�info)Z grouplistZ groupinstallZ groupupdateZ groupremoveZ grouperaseZ groupinfo)�group�groups�grpz'display, or use, the groups information�upgrade)�updateZerase)r r )�summaryr r r r r �markc C sn | j �| jj�}|r<| jjd ur4| jj�d| jj� || j_| jjd u rPd| j_| j�| jj| jj�| j_d S )Nr r )�direct_commands�get�optsZcommand�subcmd�args�insert� _CMD_ALIASES)�selfZdirect� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/group.py� _canonical6 s �zGroupCommand._canonicalc s t t| ��|� d| _d S )NF)�superr �__init__�_remark)r �cli�� __class__r r r! C s zGroupCommand.__init__c C s$ t d�}t| jj�s tj�|��d S )Nz4No group data available for configured repositories.)r �len�base�compsr � exceptionsZ CompsError)r �msgr r r � _assert_compsG s zGroupCommand._assert_compsc sT � fdd�}� � � |d u r(� jjj}n� jj�d�|��}tj�t tj� ||��S )Nc s � j jj�| j�}| S �N)r' �history�envr �id)r. Z env_found�r r r �available_predM s z7GroupCommand._environment_lists.<locals>.available_pred�,) r+ r' r( � environments�environments_by_pattern�joinr �utilZmapallr � partition)r �patternsr1 �envsr r0 r �_environment_listsL s z GroupCommand._environment_listsc sz � fdd�}g }g }� � � |d u r0� jjj}n� jj�d�|��}|D ](}|}||�r\|}|rf|jrH|�|� qH||fS )Nc s � j jj�| j�}|rdS dS )NTF)r' r- r r r/ )r Z group_foundr0 r r �installed_predZ s z1GroupCommand._group_lists.<locals>.installed_predr2 )r+ r' r( r �groups_by_patternr5 � uservisible�append) r r= r8 r; � installed� availableZgrpsr Ztgt_listr r0 r � _group_listsY s zGroupCommand._group_listsc C sr |D ]d}d}| j j�|�D ]}| j�|� d}q| j j�|�D ]}| j�|� d}q>|st�t d�|� qdg fS )NFTz!Warning: Group %s does not exist.r ) r' r( r4 �outputZ display_groups_in_environmentr< Zdisplay_pkgs_in_groups�logger�errorr )r �userlistZstrngZ group_matchedr. r r r r �_infoq s zGroupCommand._infoc sh d}d}d}| j jjp | jj�|r�|d dkr>d}|�d� q |d dkrZd}|�d� q |d dkrvd}|�d� q |d dkr�d�|�d� q q�q | jjr�d}| jjr�d}| jjr�d}|s�d }d}|d u�r4|D ]P}| j j }t |� |��dk}t |� |��dk} |s�| s�t �td �d |� d}q�|�r4dg fS | �|�\} } | �||�\} } � �fd d �}�fd d�}|�s�|td�| � |�s�|td�| � |�s�d� | D ]$}|j�r��q�|td�|� d� �q�d� | D ]$}|j�s�q�|td�|� d� �q�|�rdg fS d� | D ]$}|j�r �q |td�|� d� �q d� | D ]$}|j�sJ�q:|td�|� d� �q:dg fS )N� r �hiddenr? r@ �idsTFzWarning: No groups match:z %sc sV � s t | � dtj�|j� }�r0|d|j 7 }|jrD|d|j 7 }t d�|�� d S )N� %s� (%s)z [%s]z{})�printr r6 �_name_unset_wrapper�ui_namer/ � lang_only�format)�sectr r* ��done� print_idsr r �_out_grp� s z$GroupCommand._list.<locals>._out_grpc sF |r t | � |D ]0}dtj�|j� }� r8|d|j 7 }t |� qd S )NrJ rK )rL r r6 rM rN r/ )rQ r9 �er* )rT r r �_out_env� s z$GroupCommand._list.<locals>._out_envz Available Environment Groups:z Installed Environment Groups:�Installed Groups:�Installed Language Groups:�Available Groups:�Available Language Groups:)r' �conf�verboser rI �poprH r? r@ r( r& r< r4 rC rD r r: rA rO )r rE r= Z showinstalledZ showavailableZerrsr r( Zin_groupZin_environmentZenv_instZ env_availr? r@ rU rW r rR r �_list� s� � zGroupCommand._listc C s� t | jj| jjt jt jB t jt jB �}| j�� }|j |� }| j j rXt | jj jdg �}nt | jj j�}tj�|�}|jD ]}|�||� qx|jD ]}|�||� q�d S )N�optional)r r' r( r- �GROUPS� ENVIRONMENTSZ AVAILABLE� INSTALLED�_build_comps_solverr r � with_optional�tupler\ �group_package_types�libdnfZ transactionZlistToCompsPackageTyper3 Z_environment_installr Z_group_install) r r8 �q�solver�res�typesZ pkg_types�env_idZgroup_idr r r � _mark_install� s � zGroupCommand._mark_installc C s� t | jj| jjt jt jB t j�}| j�� }|j|� }|j D ] }t j � |�sNJ �|� |� q:|jD ] }t j � |�stJ �|�|� q`d S r, )r r' r( r- ra rb rc rd r r3 r r6 Zis_string_typeZ_environment_remover Z _group_remove)r r8 ri rj rk rm Zgrp_idr r r � _mark_remove� s � zGroupCommand._mark_removec C s* |d | j v r"|d |dd � fS d|fS )Nr rG r )� _MARK_CMDS)r �extcmdsr r r � _mark_subcmd s zGroupCommand._mark_subcmdc C s d}t |�dkr*|d dkr*d}|�d� | jjr6d}|s>d }| �||�\}}dd� }d}|D ]}|jrjq^|d7 }q^|td�|� d}|D ]}|js�q�|d7 }q�|td�|� d}|D ]}|jr�q�|d7 }q�|td �|� d}|D ]}|js�q�|d7 }q�|td �|� dg fS ) NrG r rH c S s |sd S t �d| |� d S )Nz%s %u)rC r )rQ Znumr r r rU s z'GroupCommand._summary.<locals>._out_grprX rY FrZ r[ )r& r^ r rH rA rO r )r rE r= r? r@ rU rS r r r r �_summary sH zGroupCommand._summaryc C s� | j ddtd�d� | �� }|j ddtd�d� |j ddtd�d� |j d dtd �d� |j d dtd �d� | j d ddtd��tjd d�tjdd � ��d� | j dddtd�d� d S )Nz--with-optional� store_truez$include optional packages from group)�action�helpz--hiddenzshow also hidden groupsz --installedzshow only installed groupsz --availablezshow only available groupsz--idszshow also ID of groupsr �?ZCOMMANDz'available subcommands: {} (default), {}r �, rG )�nargs�metavarrv r �*Z COMMAND_ARGz argument for group subcommand)� add_argumentr � add_mutually_exclusive_grouprP r �_GROUP_SUBCOMMANDSr5 )�parserZ grpparserr r r � set_argparser< s2 ����� �� �zGroupCommand.set_argparserc C s� | � � | jj}| jj}|| jvrBt�td�d�| j�� t j j �|dv rd|sd| j j � | � t j j �| j j}d|_|dv r�d|_d|_|dkr�d|_d|_nd|_|dvr�t�| j� |dv r�t�| j| j � d S ) Nz$Invalid groups sub-command, use: %s.rx )r r r r T)r r r r r F)r r )r r r r r~ rC Zcriticalr r5 r r# ZCliErrorZ optparser� print_help�demandsZsack_activationZ root_userZ resolvingZ allow_erasingZavailable_reposr Z_checkEnabledRepor' Z _checkGPGKey)r �cmdr r� r r r � configureP s0 � zGroupCommand.configurec C s� | j j}| j j}|dkr"| �|�S |dkr4| �|�S |dkrF| �|�S |dkr�| �|�\}}|dkrn| �|�S |dkszJ �| �|�S |dk�r4| j j r�t | j j j dg �}nt | j j j �}d| _z| j �||| j j j�W S tjj�y2 } z<td �}t�|| j jj�|�� tj�td ���W Y d }~n d }~0 0 |d k�rJ| j �|�S |dk�r�|D ]2}z| j � |g� W n tjj �y� Y n0 �qXd S ) Nr r r r r r r` TzNo package %s available.z)Unable to find a mandatory group package.r ) r r r rs r_ rF rr ro rn re rf r' r\ rg r" Zenv_group_install�strictr r) Z MarkingErrorr rC r rB ZtermZboldZPackagesNotAvailableErrorZenv_group_upgradeZenv_group_remove�Error)r r� rq r rl rV r* �argr r r �runo sJ �� zGroupCommand.runc C sb | j s d S | jj}| jj}|j}| jj�� �� j|d�D ]$}|j � |�}|� ||� ||�� q8d S )N)�name) r" r' Z_goalr- Z group_membersZsack�queryr? ZfiltermZrpmZ get_reasonZ set_reasonZ group_reason)r Zgoalr- �names�pkg�reasonr r r �run_transaction� s z GroupCommand.run_transaction) �__name__� __module__� __qualname__�__doc__r rf �keys�aliasesr r r rp r~ r r! r+ r: rA rF r_ rn ro rr rs � staticmethodr� r� r� r� � __classcell__r r r$ r r $ s> �� h / *r )Z __future__r r Z dnf.compsr Zdnf.clir Zdnf.i18nr r Zlibdnf.transactionrh r Zdnf.exceptionsZdnf.utilZloggingZ getLoggerrC ZCommandr r r r r �<module> s __pycache__/repolist.cpython-39.pyc 0000644 00000016517 15025176534 0013270 0 ustar 00 a �!cz2 � @ s� d dl mZ d dl mZ d dlmZ d dlmZmZmZm Z d dl m Z d dl Z d dlZ d dlZ d dlZd dlZd dlZd dlZe�d�Zdd � Zd d � Zd d � Zdd� ZG dd� dej�ZdS )� )�absolute_import)�unicode_literals)�commands)�_�ucd�fill_exact_width� exact_width)� OptionParserN�dnfc C sd |rt j�| j�� �ntd�}| jdkr4td�| S | jsFtd�| S t| j�}td�||f S d S )N�unknown���zNever (last: %s)zInstant (last: %s)z%s second(s) (last: %s))r �util�normalize_time�_repo� getTimestampr Zmetadata_expire� _num2ui_num)�repo�mdZlast�num� r �=/usr/lib/python3.9/site-packages/dnf/cli/commands/repolist.py� _expire_str% s r c C s t tj�d| d��S )Nz%dT)r r Zpycomp�format)r r r r r 0 s r c C sF | j �� }| j�� }|D ](}t�||�r. dS t�||�r dS qdS )NTF)�id�lower�name�fnmatch)r �patterns�ridZrnm�patr r r � _repo_match4 s r c C s: d}| j tjd�j|jd�D ]}||j7 }q tjj� |�S )Nr ��flags�Z reponame__eq) �query�hawkey�IGNORE_EXCLUDES�filtermr �_sizer �clir Z format_number)�sackr �ret�pkgr r r � _repo_size? s r- c @ s@ e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d S ) �RepoListCommandzVA class containing methods needed by the cli to execute the repolist command. )Zrepolist�repoinfoz,display the configured software repositoriesc C sx | � � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � | jddddg d�tjtd�d� d S )Nz--all� _repos_action� store_const�allzshow all repos)�dest�action�const�default�helpz --enabled�enabledz show enabled repos (default))r3 r4 r5 r7 z --disabled�disabledzshow disabled repos�repos�*�enabled-defaultZ REPOSITORY)r2 r8 r9 zRepository specification)�nargsr6 �metavar�choicesr4 r7 )� add_mutually_exclusive_group� add_argumentr r ZPkgNarrowCallback)�parserZ repolimitr r r � set_argparserN s$ ��� �z RepoListCommand.set_argparserc C s | j js | jjtjtjd� d S )N)�stdout�stderr)�opts�quietr) Zredirect_logger�loggingZWARNING�INFO)�selfr r r � pre_configure_ s z RepoListCommand.pre_configurec C sT | j js| j�� | jj}| jjjs0| j jdkr<d|_ d|_ | j j rP| j j | j _ d S )Nr/ T) rF rG r) Zredirect_repo_progress�demands�base�conf�verbose�commandZavailable_reposZsack_activationr0 � repos_action)rJ rL r r r � configurec s zRepoListCommand.configurec - C sR | j j}dd� | j jD �}| jjj}t| jj�� �}|jt � d�d� | j j }|j d |jd }|j d }|jd }d } g } |s�t�td �� d S |d kp�|d ko�|} g } |D �]�} t|�r�t| |�s�q�d \}}}d}d }d}| r�||| }}}| j�rfd}|dk�r q�| �s&|�s&| j jdk�rB|td� | }ttd��}|�sV| j jdk�r�t| jj| �}n:d}|dks�|d k�r�|�s�q�|td� | }ttd��}|�s�| j jdk�s�t| j�}| �|| j ||ff� q�|�r�| j }nd }| j � td�| j�| j � td�| j �g}| �r.|| j � td�|�g7 }|�r^| j � � �r^|| j � td�| j � � �g7 }|�r�| j �!� �r�| j �!� }|| j � td�d�"t#|���g7 }|�r | j �$� �r dd� | j �$� D �}|�%� D ]4\}}|| j � td �d |d�"t#|��f �g7 }�q�|�r�t| jjj&t'j(d �j)| jd ��} t| jj�&� j)| jd ��} t*| �} t*| �} | | 7 } || j � td �t+j,�-| j �.� ��| j � td!�| �| j � td"�| �| j � td#�|�g7 }| j/�r|| j � td$�| j/�g7 }|�r$| j �0� } || j � td%�t+j,�-| ��g7 }n"| j1�r$|| j � td&�| j1�g7 }| j2}!|!�rP|| j � td'�d�"|!��g7 }nF|�r�| j �3� }"|"�r�d(|"d t|"�d) f }#|| j � td'�|#�g7 }t4| |�}$|| j � td*�|$�g7 }| j5�r�|| j � td+�d�"| j5��g7 }| j6�r|| j � td,�d�"| j6��g7 }|�r&|| j � td-�|�g7 }| j7�rH|| j � td.�| j7�g7 }| �d/�"t8t|��� q�| �rtt9d0�"| �� |�s | �r ttd1��}%d }&d }'| D ]J\}}(\}}|%t|�k �r�t|�}%|&t|(�k �r�t|(�}&|'|k �r�|}'�q�| �r,ttd2��|'k�r|j:|%ttd2�� d3 })n|j:|%|' d3 })n|j:|%d) })|)|&k �rJ|)}&n$|)|&8 })|%|)d3 7 }%|&|)|)d3 7 }&t;td1�|%�}*| �r�t;td4�|&|&�}+ntd4�}+| �s�t9d5|*|+f � nt9d6|*|+td2�f � | D ]P\}}(\}}| �s�t9d5t;||%�|(f � �q�t9d6t;||%�t;|(|&|&�|f � �q�|�s4| j jdk�rNtd7�},t9|,�<t*| ��� d S )8Nc S s g | ] }|� � �qS r )r )�.0�xr r r � <listcomp>p � z'RepoListCommand.run.<locals>.<listcomp>r )�keyZgreenZboldZredZnormalr zNo repositories availabler2 r< )� rX rX rX Tr9 r/ r8 FzRepo-id : zRepo-name : zRepo-status : zRepo-revision : zRepo-tags : z, c S s i | ]\}}||�qS r r )rS �k�vr r r � <dictcomp>� rV z'RepoListCommand.run.<locals>.<dictcomp>zRepo-distro-tags : z[%s]: %sr! r# zRepo-updated : zRepo-pkgs : zRepo-available-pkgs: zRepo-size : zRepo-metalink : z Updated : zRepo-mirrors : zRepo-baseurl : z %s (%d more)� zRepo-expire : zRepo-exclude : zRepo-include : zRepo-excluded : zRepo-filename : � z zrepo id�status� z repo namez%s %sz%s %s %szTotal packages: {})=rF rQ r: rM rN rO �list�values�sort�operator� attrgetter�output�termZFG_COLORZMODE�loggerZwarningr �lenr r8 rP r r- r* r r �appendr �metadataZ fmtKeyValFillr Z getRevisionZgetContentTags�join�sortedZ getDistroTags�itemsr$ r% r& r' r r r r ZgetMaxTimestampZmetalinkr Z mirrorlistZbaseurlZ getMirrorsr Z excludepkgsZ includepkgsZrepofile�map�print�columnsr r )-rJ �argZextcmdsrO r: rf Z on_ehibegZ on_dhibegZon_hiendZtot_numZcolsZinclude_statusZrepoinfo_outputr ZehibegZdhibegZhiendZ ui_enabledZ ui_endis_widZui_excludes_numr8 Zui_sizer r �out�tagsZdistroTagsDictZdistror Z num_availableZui_numZui_num_available�tsZbaseurlsZmirrors�urlZexpireZid_lenZnm_lenZst_lenZrname�leftZtxt_ridZtxt_rnam�msgr r r �runn sP ��� �� ���� � �� � ��� � zRepoListCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodrC rK rR rx r r r r r. F s r. ) Z __future__r r Zdnf.clir Zdnf.i18nr r r r Zdnf.cli.option_parserr Zdnf.cli.formatr Z dnf.pycompZdnf.utilr r% rH rc Z getLoggerrg r r r r- ZCommandr. r r r r �<module> s" __pycache__/remove.cpython-39.pyc 0000644 00000007557 15025176534 0012730 0 ustar 00 a �B�f� � @ s� d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z d dl Z d dl Z d dlZe�d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)�_)� OptionParserN�dnfc @ sb e Zd ZdZejejejejejejd�Zde e� � � Z e d�Z edd� �Zdd� Zd d � Zd S ) � RemoveCommandzRemove command.)zremove-nz remove-naz remove-nevrazerase-nzerase-naz erase-nevra)�removeZeraseZrmz-remove a package or packages from your systemc C sf | � � }|jdddtd�d� |jddtjd� |jddtd �d� | jd d td �tjtd �d� d S )Nz --duplicates� store_true� duplicatedzremove duplicated packages)�action�dest�helpz --duplicated)r r z--oldinstallonlyz*remove installonly packages over the limitZpackages�*zPackage to removeZPACKAGE)�nargsr r �metavar)� add_mutually_exclusive_group� add_argumentr �argparse�SUPPRESSr ZParseSpecGroupFileCallback)�parserZmgroup� r �;/usr/lib/python3.9/site-packages/dnf/cli/commands/remove.py� set_argparser0 s" �����zRemoveCommand.set_argparserc C s^ | j j}d|_d|_d|_| jjr*d|_n0tj j rN| jj rNd|_d|_ d|_ n d|_ d|_d S )NTF)Zcli�demandsZ resolvingZ root_userZsack_activation�optsr Zavailable_reposr �base� WITH_MODULES� grp_specsZfresh_metadataZ allow_erasing)�selfr r r r � configure? s zRemoveCommand.configurec C sB g }| j j| jv r"| j| j j g}| j j| j j7 _d}| j j�r<| jj�� }| j� |� � �}|�� � |�}|s�t � td�� d S |�� �� D ]�\\}}}t|�dk r�q�|jdd� z| j�t|d �� W nF tjj�y d} td�} t �| | jjj�t|d ��| � Y n0 |d d � D ]} | j� | � �q"q�d S | j j �r�| jj�� }| j� |� � �� d �}| jj� � } | d u�r�|j | j!| j"| j#d �} | �r�|� | �}|�r�|D ]} | j� | � �q�nt � td �� d S | j j$�r|�r| j j$D ]&}td �} t �| | jjj�|�� �q�n�| j j$�r�tjj%�rftj&j'�(| j�}|�)| j j$�}t| j j$�t|�k�rnd}n| j j$}|�r�|D ]8}z| j�*|g��r�d}W n tjj+�y� Y n0 �qx| j jD ]n}z| jj)||d� W nN tjj,�y } z0d�-|j.| jjj�|��} t � | � W Y d }~nd }~0 0 d}�q�|�s>t �td�� d S )NFz)No duplicated packages found for removal.� T)�reverser � z%Installed package %s%s not available.� ���)�epoch�version�releasez.No old installonly packages found for removal.zNot a valid form: %s)�formsz{}: {}z No packages marked for removal.)/r Zcommand� nevra_formsZ pkg_specs� filenamesr r Zsack�queryZ_get_installonly_queryZ installed� difference�logger�infor Z_na_dict�items�len�sortZ reinstall�strr � exceptionsZPackagesNotAvailableErrorZwarning�outputZtermZboldZpackage_removeZoldinstallonlyZlatestZget_running_kernel�filterr& r' r( r r �module� module_baseZ ModuleBaser Zenv_group_remove�ErrorZ MarkingError�format�value)r r) Zdone�qZinstonlyZdups�nameZarchZ pkgs_listZxmsg�msg�pkgZkernelZrunning_installonlyZgrp_specr8 Z skipped_grps�groupZpkg_spec�er r r �runP s� ( � zRemoveCommand.runN)�__name__� __module__� __qualname__�__doc__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr* �tuple�keys�aliasesr Zsummary� staticmethodr r rB r r r r r # s � r )Z __future__r r Zdnf.clir Zdnf.i18nr Zdnf.cli.option_parserr Zdnf.baser r rG Zdnf.exceptionsZloggingZ getLoggerr. ZCommandr r r r r �<module> s __pycache__/clean.cpython-39.opt-1.pyc 0000644 00000010301 15025176534 0013431 0 ustar 00 a �!cn � @ s� d dl mZ d dl mZ d dlmZ d dlmZmZ d dlm Z d dlZ d dl Z d dl Z d dl Z d dlZ d dlZd dlZd dlZd dlZe�d�Zg d�d gd gd gg d �d �Zdd� Zdd� Zdd� Zdd� ZG dd� dej�ZdS )� )�absolute_import)�unicode_literals)�commands)�_�P_)�miscN�dnf)�metadata�dbcache� expire-cache�packagesr r )r r r )r r r r �allc c sN t �| �D ]>\}}}t j�|| �}|D ] }t j�||�}t j�|�V q&q dS )z:Traverse dirpath recursively and yield relative filenames.N)�os�walk�path�relpath�join�normpath)�dirpath�root�dirs�files�base�fr � r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/clean.py�_tree1 s r c s � fdd�| D �S )z5Yield those filenames that match any of the patterns.c 3 s( | ] }� D ]}t �||�r |V q qd S �N��re�match)�.0r �p��patternsr r � <genexpr>< � z_filter.<locals>.<genexpr>r )r r$ r r# r �_filter: s r' c C sH d}|D ]:}t j�| |�}t�tjjtd�|� t � |� |d7 }q|S )z(Remove the given filenames from dirpath.r zRemoving file %s� ) r r r �logger�logr �loggingZDDEBUGr r Zunlink_f)r r �countr r r r r �_clean? s r- c s0 t jjd � � fdd�| D �}tdd� |D ��S )z:Return the repo IDs that have some cached metadata around.r c 3 s | ]}t �� |�V qd S r r )r! r �Zmetapatr r r% M r& z _cached_repos.<locals>.<genexpr>c s s | ]}|r|� d �V qdS )ZrepoidN)�group)r! �mr r r r% N r& )r �repo� CACHE_FILES�set)r �matchesr r. r � _cached_reposJ s r5 c @ s0 e Zd ZdZdZed�Zedd� �Zdd� Z dS ) � CleanCommandzSA class containing methods needed by the cli to execute the clean command. )Zcleanzremove cached datac C s | j ddt�� td�d� d S )N�type�+zMetadata type to clean)�nargs�choices�help)� add_argument� _CACHE_TYPES�keysr )�parserr r r � set_argparserY s �zCleanCommand.set_argparserc C s� | j jj}tj�|d�}tj�|d�}tj�| j jjd�}�z<|�� |� �� |�� t dd� | j j D ��}t t |��}t�tdd�|� �� d|v r�t|�}| j jj�|� |�d� t�td�� dd � |D �}t|t||��} t�td d | �| � W d � W d � W d � W d S 1 �s,0 Y W d � n1 �sL0 Y W d � n1 �sl0 Y W q: tjj �y� } z@| j jj �s�td �| j } t�| � t � d � n| �W Y d } ~ q:d } ~ 0 0 q:d S )NTc s s | ]}t | D ] }|V qqd S r )r= )r! �c�tr r r r% g r& z#CleanCommand.run.<locals>.<genexpr>zCleaning data: � r zCache was expiredc S s g | ]}t jj| �qS r )r r1 r2 )r! rB r r r � <listcomp>q r& z$CleanCommand.run.<locals>.<listcomp>z%d file removedz%d files removedz*Waiting for process with pid %d to finish.� )!r Zconf�cachedirr �lockZbuild_metadata_lockZbuild_download_lockZbuild_rpmdb_lockZ persistdirr3 Zoptsr7 �listr r) �debugr r r5 Z_repo_persistorZexpired_to_add�update�remove�infor- r' r � exceptionsZ LockErrorZ exit_on_lock�pid�time�sleep) �selfrF Zmd_lockZ download_lockZ rpmdb_lock�typesr Zexpiredr$ r, �e�msgr r r �run_ s0 � zCleanCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr@ rU r r r r r6 Q s r6 ) Z __future__r r Zdnf.clir Zdnf.i18nr r Zdnf.yumr r Zdnf.exceptionsZdnf.lockZ dnf.loggingZdnf.repor+ r r rO Z getLoggerr) r= r r' r- r5 ZCommandr6 r r r r �<module> s2 � __pycache__/reinstall.cpython-39.pyc 0000644 00000005775 15025176534 0013430 0 ustar 00 a �!c] � @ sl d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z e � d�Z G dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_N�dnfc @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �ReinstallCommandzSA class containing methods needed by the cli to execute the reinstall command. )� reinstallZreizreinstall a packagec C s" | j ddtd�tjtd�d� d S )N�packages�+zPackage to reinstallZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/reinstall.py� set_argparser( s �z ReinstallCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� dS )a Verify that conditions are met so that this command can run. These include that the program is being run by the root user, that there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. TN) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure. s zReinstallCommand.configurec C s� d}| j j| jjd| j jjd�D ]N}z| j �|� W n4 tjj yh t � t d�| j jj �|j�� Y q 0 d}q | jjdd� | jjD � D �].}z| j �|� W �n tjj�y } zT|jD ]$}t � t d�| jj �|j�� q�q�t � t d�| j jj �|�� W Y d }~q�d }~0 tjj�y� } z`|jD ]L}d}| j j�|�}|�rVt d �| }t d �}t � || j jj �|�|� �q.W Y d }~q�d }~0 tjj �y� d�s�J d ��Y q�0 d}q�|�s�tj�t d ���d S ) NF)�strict�progresszNo match for argument: %sTc S s g | ] }d | �qS )�@r )�.0�xr r r � <listcomp>L � z(ReinstallCommand.run.<locals>.<listcomp>z(Package %s available, but not installed.� z (from %s)z%Installed package %s%s not available.z+Only the above marking errors are expected.z!No packages marked for reinstall.)r Zadd_remote_rpmsr r �outputr Zpackage_reinstallr � exceptionsZ MarkingError�logger�infor ZtermZbold�locationZ pkg_specsZ grp_specsr ZPackagesNotInstalledErrorr �nameZPackagesNotAvailableError�historyZrepo�Error)r Zdone�pkgZpkg_spec�errZxmsgZpkgrepo�msgr r r �run= sL � � � � � zReinstallCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r. r r r r r ! s r )Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.exceptionsr ZloggingZ getLoggerr% ZCommandr r r r r �<module> s __pycache__/distrosync.cpython-39.opt-1.pyc 0000644 00000002705 15025176534 0014561 0 ustar 00 a �!c� � @ s: d dl mZ d dlmZ d dlmZ G dd� dej�ZdS )� )�absolute_import)�commands)�_c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �DistroSyncCommandzZA class containing methods needed by the cli to execute the distro-synch command. )z distro-syncZ distrosyncz distribution-synchronizationZdsyncz?synchronize installed packages to the latest available versionsc C s | j ddtd�d� d S )N�package�*zPackage to synchronize)�nargs�help)� add_argumentr )�parser� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/distrosync.py� set_argparser" s z DistroSyncCommand.set_argparserc C sF | j j}d|_d|_d|_d|_t�| j| j � t� | j| j j � d S )NT) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�baseZ_checkEnabledRepo�optsr )�selfr r r r � configure& s zDistroSyncCommand.configurec C s | j �| jj�S )N)r Zdistro_sync_userlistr r )r r r r �run/ s zDistroSyncCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r r r r r r s r N)Z __future__r Zdnf.clir Zdnf.i18nr ZCommandr r r r r �<module> s __pycache__/repoquery.cpython-39.opt-1.pyc 0000644 00000053547 15025176534 0014425 0 ustar 00 a �B�f � � @ s d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ d dlm Z d dl Z d dl Z d dl Z d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZe �d�Zd Ze �d �Zd Zd d dddddddd� Zdd� ZG dd� de j �Z G dd� dej �Z G d d � d e �Z!dS ) � )�absolute_import)�print_function)�unicode_literals)�_)�commands)� OptionParserN�dnfz.%{name}-%{epoch}:%{version}-%{release}.%{arch}z%(-?\d*?){([:.\w]+?)}a name, arch, epoch, version, release, reponame (repoid), from_repo, evr, debug_name, source_name, source_debug_name, installtime, buildtime, size, downloadsize, installsize, provides, requires, obsoletes, conflicts, sourcerpm, description, summary, license, url, reason� conflicts�enhances� obsoletes�provides� recommends�requiresZ requires_pre�suggests� supplements) r r r r r r � requires-prer r c C s� dd� }dd� }| � dd�� dd�} t�� D ]\}}| � ||�} q,d }d }t�| �D ]0}||| ||�� � �7 }|||�7 }|�� }qT||| |d � �7 }|S ) z:Convert a rpm like QUERYFMT to an python .format() string.c S s^ | � � d }| � � d }|rJ|d dkr:d|dd � }nd| }d| }d|�� | d S ) Nr � �-�>�<�:z{0.�})�groups�lower)Zmatchobj�fill�key� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/repoquery.py�fmt_replD s z rpm2py_format.<locals>.fmt_replc S s | � dd�� dd�S )N�{z{{r z}})�replace)Ztxtr r r �bracketsO s z rpm2py_format.<locals>.bracketsz\n� z\t� � r N)r � OPTS_MAPPING�items� QFORMAT_MATCH�finditer�start�end)� queryformatr r! r �valueZfmt�spos�itemr r r � rpm2py_formatB s r/ c @ s e Zd ZdZdS )�_CommaSplitCallbackz\s*,\s*N)�__name__� __module__� __qualname__ZSPLITTERr r r r r0 _ s r0 c @ s� e Zd ZdZejejejd�Zde e� � � Z e d�Z edd� �Zedd� �Zd d � Zd d � Zd d� Zdd� Zd$dd�Zd%dd�Zd&dd�Zdd� Zdd � Zd d � Zd d � Zd'd"d#�Z dS )(�RepoQueryCommandzSA class containing methods needed by the cli to execute the repoquery command. )z repoquery-nz repoquery-nazrepoquery-nevra)Z repoqueryZrqz$search for packages matching keywordc C s, | j r|j| j d� | jr(|j| jd� |S )z'Filter query by repoid and arch options)Zreponame)�arch)Zrepo�filterm�arches)�opts�queryr r r �filter_repo_archm s z!RepoQueryCommand.filter_repo_archc C s� | j ddddtd�d� | j ddtd�d � | j d d d g td td�d� | j ddddtd�d� | j dg tdtd�d� | j dg tdtd�d� | j d g tdtd �d� | j d g tdtd �d� | j d g tdtd!�d� | j d"g tdtd#�d� | j d$g tdtd%�d� | j d&g tdtd'�d� | j d(g tdtd)�d� | �� }|j d*dtd+�d � |j d,dtd-�d � | j d.dtd/�d � | j d0dtd1�d � | j d2dtd3�d � | j d4dtd5�d � | j d6dtd7�d � | j d8d9ttd:�d;� | j d<dtd=�d � | �� }|j d>d?d@dAdtdB�dC� |j dDdEdFdAdtdG�dC� |j dHdIdJdAdtdK�dC� |j dLdMdAdtdN�dC� |j dOdPdQttdR�dS� | j dTdtdU�d � |j dVdQtdWtdX�dY� |j dZdQd[dWtd\�dY� |j d]dQd^dWtd_�dY� |j d`dtda�d � | �� }|j dbdcdddWtde�dY� |j dfdcdddWtjdY� |j dgdcdhdWtdi�dY� |j djdcdkdWtdl�dY� | j dmdtdn�d � | �� }tdo�tdp�tdq�tdr�tds�tdt�tdu�tdv�tdw�dx� }|�� D ]&\}}dy| }|j |dzdW||d{� �q�| j d|dtd}�d � td~�td�td��td��j t j j d��td��d��} | �� } | �� D ]&\} } dy| } | j | d�dW| | d{� �q.| j d�d�dWd�tjd{� | j d�dtd��d � | j d�d�d�td��d�� d S )�Nz-az--allZqueryall� store_truezNQuery all packages (shorthand for repoquery '*' or repoquery without argument))�dest�action�helpz--show-duplicatesz(Query all versions of packages (default))r= r> z--archz --archlistr7 z[arch]z show only results from this ARCH)r< �defaultr= �metavarr> z-fz--file�FILE�+z show only results that owns FILE)r@ �nargsr> z--whatconflictsZREQz#show only results that conflict REQ)r? r= r@ r> z --whatdependszjshows results that requires, suggests, supplements, enhances, or recommends package provides and files REQz--whatobsoletesz#show only results that obsolete REQz--whatprovidesz"show only results that provide REQz--whatrequiresz:shows results that requires package provides and files REQz--whatrecommendsz$show only results that recommend REQz--whatenhancesz"show only results that enhance REQz--whatsuggestsz"show only results that suggest REQz--whatsupplementsz%show only results that supplement REQ� --alldepsz=check non-explicit dependencies (files and Provides); default� --exactdepsz:check dependencies exactly as given, opposite of --alldeps� --recursivezOused with --whatrequires, and --requires --resolve, query packages recursively.z --deplistz>show a list of all dependencies and what packages provide themz --resolvez.resolve capabilities to originating package(s)z--treez"show recursive tree for package(s)z--srpmz#operate on corresponding source RPMz--latest-limit� latest_limitzOshow N latest packages for a given name.arch (or latest but N if N is negative))r< �typer> z--disable-modular-filteringz-list also packages of inactive module streamsz-iz--info� queryinfoFz+show detailed information about the package)r< r? r= r> z-lz--list� queryfilelistz!show list of files in the packagez-sz--source�querysourcerpmz show package source RPM namez --changelogs�querychangelogsz show changelogs of the packagez--qfz --queryformatr+ zfdisplay format for listing packages: "%%{name} %%{version} ...", use --querytags to view full tag list)r< r? r> z --querytagsz-show available tags to use with --queryformatz--nevra� store_constzZuse name-epoch:version-release.architecture format for displaying found packages (default))r< �constr= r> z--nvrz %{name}-%{version}-%{release}zQuse name-version-release format for displaying found packages (rpm query default)z--envraz.%{epoch}:%{name}-%{version}-%{release}.%{arch}zPuse epoch:name-version-release.architecture format for displaying found packagesz --groupmemberz=Display in which comps groups are presented selected packagesz --duplicates� pkgfilter� duplicatedz/limit the query to installed duplicate packagesz --duplicatedz --installonly� installonlyz1limit the query to installed installonly packagesz --unsatisfied� unsatisfiedzClimit the query to installed packages with unsatisfied dependenciesz --locationz5show a location from where packages can be downloadedz5Display capabilities that the package conflicts with.zaDisplay capabilities that the package can depend on, enhance, recommend, suggest, and supplement.z2Display capabilities that the package can enhance.z-Display capabilities provided by the package.z1Display capabilities that the package recommends.z1Display capabilities that the package depends on.z�If the package is not installed display capabilities that it depends on for running %%pre and %%post scriptlets. If the package is installed display capabilities that is depends for %%pre, %%post, %%preun and %%postun.z/Display capabilities that the package suggests.z5Display capabilities that the package can supplement.) r �dependsr r r r r r r z--%s� packageatr)r< r= rN r> � --availablez Display only available packages.z Display only installed packages.zLDisplay only packages that are not present in any of available repositories.zQDisplay only packages that provide an upgrade for some already installed package.zIDisplay only packages that can be removed by "{prog} autoremove" command.��progz2Display only packages that were installed by user.)� installed�extrasZupgrades�unneeded� userinstalled�listz --autoremoverZ z--recentz%Display only recently edited packagesr �*ZKEYzthe key to search for)rC r@ r> ) � add_argumentr r0 � add_mutually_exclusive_group�int�QFORMAT_DEFAULT�argparse�SUPPRESSr& �formatr �util� MAIN_PROG)�parserZwhatrequiresformZoutformrO Zpackage_attributeZ help_msgs�argZhelp_msg�nameZ help_listZ list_groupZlist_argZhelp_argZswitchr r r � set_argparserv s^ �� � � � � � � � � � � ��� � ���� �� � � �� �� � �� ������� � �� � � �z RepoQueryCommand.set_argparserc C s | j js | jjtjtjd� d S )N)�stdout�stderr)r8 �quiet�cliZredirect_logger�loggingZWARNING�INFO��selfr r r � pre_configure s z RepoQueryCommand.pre_configurec C sj | j js| j�� | jj}| j jrJ| j jrB| j�dd| j j � nd| j _| j jrVd S | j j rv| j jsvt j� t d���| j j r�| j jr�| j�dd� t| j j| j jdko�| j j g�s�t j� t d���| j js�| j j�r| j j�s| j j�st j� t d �| j j�rd nd����| j j�r$| jj�� | j jd v�r@| j jd k�sJ| j j�rPd |_d |_ | j j �rfd |_ d S )Nz --obsoletes�--r z�Option '--resolve' has to be used together with one of the '--conflicts', '--depends', '--enhances', '--provides', '--recommends', '--requires', '--requires-pre', '--suggests' or '--supplements' optionsrF rE r z�Option '--recursive' has to be used with '--whatrequires <REQ>' (optionally with '--alldeps', but not with '--exactdeps'), or with '--requires <REQ> --resolve'z;argument {} requires --whatrequires or --whatdepends optionrD )rX r[ rQ T) r8 rm rn Zredirect_repo_progress�demandsr rT Z_option_conflict� querytags�resolver ZCliErrorr � recursive� exactdeps�any� whatrequires�alldeps� whatdependsrd �srpm�baseZreposZenable_source_reposr\ rO � availableZavailable_reposZsack_activationrL � changelogs)rr ru r r r � configure sP ����� �� zRepoQueryCommand.configurec C s |j rlg }|�dt|� � |jD ]>}|d }|�d|�d�tj�|d �tj�|d �f � q"d�|�S zpt |�}|j r�| j j � |�W S |jr�|j}|s�ttd��|�tjd � |W S |jr�|jW S t|j��|�W S W n6 t�y } z tj�t|���W Y d }~n d }~0 0 d S ) NzChangelog for %s� timestampz * %s %s %s z %a %b %d %YZauthor�textr" z Package {} contains no files)�file) rL �append�strr� �strftimer �i18n�ucd�join�PackageWrapperrI r �outputZ infoOutputrJ �files�printr rd �sysrl rK Z sourcerpmr/ r+ �AttributeError� exceptions�Error) rr r8 �pkg�outZchlog�dtZpoZfilelist�er r r �build_format_fnG s0 � z RepoQueryCommand.build_format_fnc C sJ | j j�� jdd�}|D ],}|�|�tj�|�j | j jddd���}q|S )NT��emptyF)� with_provides�with_filenames) r �sackr9 r6 �union� intersectionr �subject�Subject�get_best_query)rr ZnevrasZ base_query�resolved_nevras_queryZnevrar r r �_resolve_nevrasc s ��z RepoQueryCommand._resolve_nevrasNc C sD |r|n|}|j |d�}|�|�}|�|�}|r@| j|||d�}|S )N�r ��done)�filter� differencer� �_do_recursive_deps)rr �query_in� query_selectr� Zquery_requiredr r r r� p s z#RepoQueryCommand._do_recursive_depsFc C s� | � ||�}|j|d�}|�|j|d��}|r�|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d ��}|�|j|d ��}| jjr�| �||�}|S ) N�Zrequires__globr� �Zrecommends__glob�Zenhances__glob�Zsupplements__glob�Zsuggests__glob�r �r �r �r )r� r� r� r8 rx r� )rr �namesr9 Z all_dep_typesr� Zdepqueryr r r � by_all_deps} s z RepoQueryCommand.by_all_depsc C s� |r|n| j j�� jdd�}| j j�� jdd�}|�� D ]}|�|j|jd��}q8|�|�}|rv| j |||�|�d�}|�|�S )NTr� �r r� ) r r� r9 r6 �runr� r� r r� � _get_recursive_providers_query)rr r� � providersr� �tr� r� r r r r� � s z/RepoQueryCommand._get_recursive_providers_queryc C st g }g }| j jD ]B}tjj�|�d }|�d�r<|�|� q|r|dv r|�|� q|rp| jj|d| jj j d�}|S )Nr z.rpm)�http�ftpr� �httpsF)�strict�progress) r8 r r Zpycomp�urlparse�endswithr� r Zadd_remote_rpmsr� r� )rr Zrpmnames�remote_packagesr Zschemesr r r �_add_add_remote_packages� s �z)RepoQueryCommand._add_add_remote_packagesc C s� | j jrtt� d S | j�| j � | jjj| j j r8t j nt j d�}| j j r�| �� }ddi}| j j| jv r�| j| j j g|d< d|d< g }|jdd�}|r�|�| jj�� j|d��}| j j D ]0}|�tjj|dd �j| jjfd |i|���}q�|}| j j�r |�| jjj�}| j j�r^| j j �r�| j j d k�r�t| jj � � � tj � t!d �"d d| j j ����nH| j j dk�r~|�#| jj$j%�}n(| j j �r�| j j dk�r�t&|| j j �� }| j j'dk�r�| j�(|�}|�)|��*� }n�| j j'dk�r�| j�(|�}n�| j j'dk�r\tj�+| j�}|�,| jjj-| jjj.� tj/�0|�} d| _1| j2dd�} | �sXttj3�4| �5� �� d S | j j �sn|�� }| �6| j |�}|} | j j7�r�|j| j j7d� | j j8�r�|j| j j8d�} | �|j| �9| j j8|�d��}| j j:�r�|j| j j:d� | j j;�r"|j| j j;d�} | �r| }n|j| j j;d� | j j<�rX| j j=�rH|j| j j<d� n| �>| j j<|�}| j j?�r�| j j=�r�|j| j j?d�}|�|j| j j?d��}|�|j| j j?d ��}|�|j| j j?d ��}|�|j| j j?d ��}n| �>| j j?|d�}| j j@�r |j| j j@d�} | �|j| �9| j j@|�d ��}| j jA�rX|j| j jAd �} | �|j| �9| j jA|�d ��}| j jB�r�|j| j jBd �} | �|j| �9| j jB|�d!��}| j jC�r�|j| j jCd �} | �|j| �9| j jC|�d"��}| j jD�r�|�E| j jD�}| jjF|dd#�}| j jG�rTg }|D ]<}|jH}|d u�r| jj�� j||jId$d%�}||�2� 7 }�q| jj�� j|d�}| j jJ�r�| j j<�s�| j jKd&v�r�tj � t!d'�j"tj3jLd(���| �M|| | j � d S tN� }| j jK�r�tN� } |�2� D ]p}| j j dk�s�| jj$�O|��r�| j jKd)k�r| �P|jQ|jR |jS |jT |jU � n| �Pt&|tV| j jK �� �q�| j jW�r�| j j d k�rf| �6| j | jj�� �}n| �6| j | jj�� �� �}|j| d*�}| j jX�r�|�| �Y||��}tN� }|�E� �2� D ]}|�Z| �[| j |�� �q�n|�Pd+d,� | D �� �n�| j j\�r*|�2� D ]"}|�]� }|d u�r|�Z|� �q�nf| j j^�r6g }t_tN|�2� ��D ]�}| j j dk�sj| jj$�O|��rHg }|�`d-ta|� � t_d.d/� |jQD ��D ]t}|�`d0| � tj�|�}|�| jj�}| �6| j |�� �}| j jb�s�|�E� }|�2� D ]}|�`d1ta|� � �q�q�|�`d2�c|�� �qH|�r2td3�c|�� d S | j jd�rN| �e|� d S |�2� D ]8}| j j dk�sx| jj$�O|��rV|�Z| �[| j |�� �qV|�r�| j jf�r�td3�ct_|��� ntd2�ct_|��� d S )4N)�flagsr� FZformsr� Tr� �r� )Z ignore_caser9 rX z)argument {}: not allowed with argument {}rU rt rZ r[ rP rQ rR )Zverify)Z file__glob)Zconflicts__glob)r )r )Zprovides__globr� r� r� r� r� r� r� r� r� )Zwarning�src)ri �evrr5 )r r r r r r r r z�No valid switch specified usage: {prog} repoquery [--conflicts|--enhances|--obsoletes|--provides|--recommends|--requires|--suggest|--supplements|--whatrequires] [key] [--tree] description: For the given packages print a tree of the packages.rV rS r� c s s | ]}t |�V qd S �N�r� )�.0�relr r r � <genexpr>R � z'RepoQueryCommand.run.<locals>.<genexpr>z package: c S s g | ] }t |��qS r r� )r� �reqr r r � <listcomp>^ r� z(RepoQueryCommand.run.<locals>.<listcomp>z dependency: z provider: r" z )gr8 rv r� � QUERY_TAGSrn Z _populate_update_security_filterr r� r9 Zdisable_modular_filtering�hawkeyZIGNORE_MODULAR_EXCLUDESZAPPLY_EXCLUDESr r� Zcommand� nevra_formsr� r� r6 r r� r� r� ZrecentZ_recentZconfr� r\ Z optparser� print_usager� r� r rd Z _unneeded�historyZswdb�getattrrO Z_get_installonly_queryr� rP Z rpmdb_sackZ _configureZinstallonlypkgsZinstallonly_limit�goalZGoalZprotect_running_kernelr� re Z_format_resolve_problemsZ problem_rulesr: r� Z whatconflictsr� Z whatobsoletesZ whatprovidesr{ ry r� r} ZwhatrecommendsZ whatenhancesZwhatsupplementsZ whatsuggestsrG ZlatestZ_merge_update_filtersr~ Z source_namer� ZtreerT rf � tree_seed�setZuser_installed�updater r r r r r% rw rx r� �addr� �locationZremote_locationZdeplist�sortedr� r� �verboser� Z groupmember�_group_member_reportrI )rr �qr� ZkwarkZpkgsZ query_resultsr rQ Zrpmdbr� ZsolvedZorqueryZrelsZquery_for_provideZ dependsqueryZpkg_listr� �srcnameZ tmp_queryr9 r� r� Zdeplist_outputr� r� Zproviderr r r r� � sj �� � ���� � � �� �� � � zRepoQueryCommand.runc C s i }| j jjD ] }tdd� |�� D ��||j< qi }g }|�� D ]b}g }|�� D ] \}} |j| v rP|� |� qP|r�|� d� t |��g �� t |�� q@|� t |�� q@g } t |�� �D ]4\} } | � d� t | �t dd� | �d�D �� �� q�| � d� t |��� | �rtd� | �� d S )Nc S s g | ] }|j �qS r �ri )r� r� r r r r� ~ r� z9RepoQueryCommand._group_member_report.<locals>.<listcomp>�$r" c S s g | ] }d | �qS )z @r )r� �idr r r r� � r� )r �compsr r� Z packages_iterr� r� r& ri r� � setdefaultr� r� r� �splitr� ) rr r9 Zpackage_conf_dict�groupZgroup_package_dictZpkg_not_in_groupr� Z group_id_listZgroup_idZpackage_name_setr� r Z package_listr r r r� { s2 ��$�z%RepoQueryCommand._group_member_reportc C s� | � ||�}|dkr t|� d S d}td|�D ] }|d7 }q.g }|jD ]}|�t|�� qFdtt|�� d d�|� d } t|d | d | � d S ) N���� r z| �[z: z, �]z\_ )r� r� �ranger r� r� �lenr� ) rr �levelr� r8 Z pkg_stringZspacing�xr Z requirepkgZreqstrr r r � grow_tree� s "zRepoQueryCommand.grow_treer� c C s, t t|�� �dd� d�D �] }|d u s.|dkr4t� n|}|j�d�sP|j�d�rV d S | �|||� ||vr|�|� |jr�t||j�}i }t|�D ]6} | j j � � j | d�} | D ]} | || jd | j < q�q�| j j � � j t|�� �d �} n&|j�r| �|jf|�n |j|jd �} | �| |||d |� qd S ) Nc S s | j S r� r� )�pr r r �<lambda>� r� z,RepoQueryCommand.tree_seed.<locals>.<lambda>)r r� ZrpmlibZsolvabler� �.r� r� r )r� r� r� ri � startswithr� r� rT r� r r� r9 r6 r5 r\ �valuesr| r� r� r� ) rr r9 Zaqueryr8 r� Zusedpkgsr� Zstrpkg�arri ZpkgqueryZquerypkgr r r r� � s& �zRepoQueryCommand.tree_seed)N)F)N)r� N) r1 r2 r3 �__doc__r� Z FORM_NAMEZFORM_NAZ FORM_NEVRAr� �tuple�keys�aliasesr Zsummary� staticmethodr: rj rs r� r� r� r� r� r� r� r� r� r� r� r r r r r4 c s2 � 0 Ir4 c @ sD e Zd ZdZdd� Zdd� Zedd� �Zedd � �Z ed d � �Z d S ) r� z>Wrapper for dnf.package.Package, so we can control formatting.c C s || _ d S r� )�_pkg)rr r� r r r �__init__� s zPackageWrapper.__init__c C sF t | j|�}|d u rdS t|t�r:d�tdd� |D ���S tj�|�S )Nz(none)r" c S s h | ]}t j�|��qS r )r r� r� )r� Zreldepr r r � <setcomp>� r� z-PackageWrapper.__getattr__.<locals>.<setcomp>) r� r� � isinstancer\ r� r� r r� r� )rr �attrZatrr r r � __getattr__� s zPackageWrapper.__getattr__c C s& | dkr t j �| �}|�d�S dS d S )Nr z%Y-%m-%d %H:%Mr$ )�datetimeZutcfromtimestampr� )r� r� r r r �_get_timestamp� s z PackageWrapper._get_timestampc C s | � | jj�S r� )r r� � buildtimerq r r r r � s zPackageWrapper.buildtimec C s | � | jj�S r� )r r� � installtimerq r r r r � s zPackageWrapper.installtimeN) r1 r2 r3 r� r� r r� r �propertyr r r r r r r� � s r� )"Z __future__r r r Zdnf.i18nr Zdnf.clir Zdnf.cli.option_parserr rb r ro �rer� r Zdnf.exceptionsZ dnf.subjectZdnf.utilr� Z getLogger�loggerra �compiler' r� r% r/ Z_SplitCallbackr0 ZCommandr4 �objectr� r r r r �<module> sL � X __pycache__/alias.cpython-39.pyc 0000644 00000012436 15025176534 0012514 0 ustar 00 a �!c� � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlZd dlZd dl Zd dlm Z d dl Zd dl Zd dl mZ e�d�ZG dd � d e j�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�commands)�_�dnfc @ sl e Zd ZdZed�Zedd� �Zdd� Zdd� Z d d � Z d d � Z d d� Z dd� Z dd� Zdd� Zdd� ZdS )� AliasCommand)�aliasz List or create command aliasesc C sj | � � }|jdddtd�d� |jdddtd�d� | jdd d g d �td �d � | jdddtd�d� d S )Nz--enable-resolvingF� store_truezenable aliases resolving)�default�action�helpz--disable-resolvingzdisable aliases resolving� subcommand�?�list)�addr �deletezaction to do with aliases)�nargsr �choicesr r �*zcommand[=result]zalias definition)r �metavarr )� add_mutually_exclusive_group� add_argumentr )�parserZ enable_group� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/alias.py� set_argparser* s �� � �zAliasCommand.set_argparserc C sH | j j}| jjdv rd|_tj j�� | _| j� � | jj | _ | � � d S )N)r r T) �cli�demands�optsr Z root_userr �aliasesZAliases� aliases_baseZ _load_aliases�enabledZresolving_enabled�_update_config_from_options)�selfr r r r � configure9 s zAliasCommand.configurec C s� d }| j jr d}t�td�� | j jr8d}t�td�� |d ur�tj�t j j j �sft t j j j d��� t jj�t j j j dd d|i� | j�� s�|| j_d S )NTzAliases are now enabledFzAliases are now disabled�w�mainr" )r Zenable_resolving�logger�infor Zdisable_resolving�os�path�existsr r r ZALIASES_CONF_PATH�open�close�confZ BaseConfigZwrite_raw_configfiler! Z_disabled_by_environr" )r$ r" r r r r# B s � z(AliasCommand._update_config_from_optionsc C s� i }| j jD ]�}|�dd�}|d �� }t|�� �dkrJt�td�|� q |�d�rft�td�|� q t|�dkr�t�td�|� q |d �� ||< q |S )N�=� r zInvalid alias key: %s�-z Alias argument has no value: %s) r r �split�strip�lenr( �warningr � startswith)r$ Z new_aliasesr �cmdr r r �_parse_option_aliasT s z AliasCommand._parse_option_aliasc C s~ t j�tjjj�s&ttjjjd��� ztjj� tjjj�}W n: tj j yx } z t � td�|� W Y d }~d S d }~0 0 |S )Nr& zConfig error: %s)r* r+ r, r r r �ALIASES_USER_PATHr- r. Z AliasesConfig� exceptionsZ ConfigErrorr( r6 r )r$ r/ �er r r �_load_user_aliasese s �z AliasCommand._load_user_aliasesc C s` t tjjjd�}d}|d�|�7 }|d7 }|�� D ] \}}|d�|d�|��7 }q2|�|� d S )Nr& z[main] zenabled = {} z [aliases] z{} = {} � ) r- r r r r: �format�items�join�write)r$ � user_aliasesr" Zfileobj�output�key�valuer r r �_store_user_aliasesp s z AliasCommand._store_user_aliasesc C sP | � � }|j}|d u rd S |�|� | �||j� t�td�d�|� � �� d S )NzAliases added: %s�, ) r= r �updaterG r" r( r) r rA �keys)r$ r r/ rC r r r � add_aliasesy s zAliasCommand.add_aliasesc C s� | � � }|j}|d u rd S g }|D ]<}z||= |�|� W q" ty\ t�td�|� Y q"0 q"| �||j� t�td�d� |�� d S )NzAlias not found: %szAliases deleted: %srH ) r= r �append�KeyErrorr( r) r rG r" rA )r$ �cmdsr/ rC Z valid_cmdsr8 r r r �remove_aliases� s zAliasCommand.remove_aliasesc C s� |g}z| j �|�}W nJ tjjy` } z.t�td�||d�| j j | �� W Y d }~n$d }~0 0 t td�|d�|�f � d S )Nz%s, alias %s="%s"r> z Alias %s='%s') r! �_resolver r; �Errorr( �errorr rA r �print)r$ r8 �argsr< r r r � list_alias� s �zAliasCommand.list_aliasc C s� | j jst�td�� | jjdkrL| �� }|s>tj � td���| � |� d S | jjdkr�| jj }|g krxtj � td���| � |� d S | jj s�| j js�t�td�� d S | j jD ]}| �|� q�n8| jj D ].}|| j jvr�t�td�| � q�| �|� q�d S )Nz Aliases resolving is disabled.r zNo aliases specified.r zNo alias specified.zNo aliases defined.zNo match for alias: %s)r! r" r( r6 r r r r9 r r; rQ rK r rO r r) rU )r$ r rN r8 r r r �run� s2 zAliasCommand.runN)�__name__� __module__� __qualname__r r Zsummary� staticmethodr r% r# r9 r= rG rK rO rU rV r r r r r & s r )Z __future__r r r ZloggingZos.pathr* Zdnf.clir Zdnf.cli.aliasesr Zdnf.confZdnf.exceptionsZdnf.i18nr Z getLoggerr( ZCommandr r r r r �<module> s __pycache__/upgrade.cpython-39.pyc 0000644 00000007174 15025176534 0013055 0 ustar 00 a �!c~ � @ st d dl mZ d dl mZ d dlZd dlZd dlZd dlmZ d dl m Z d dl m Z e� d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literalsN)�commands)� OptionParser)�_�dnfc @ sX e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dd� ZdS )�UpgradeCommandzTA class containing methods needed by the cli to execute the update command. )�upgrade�updatez upgrade-toz update-toZ localupdateZupz,upgrade a package or packages on your systemc C s" | j ddtd�tjtd�d� d S )NZpackages�*zPackage to upgradeZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/upgrade.py� set_argparser* s �z UpgradeCommand.set_argparserc C sZ | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� d| _ d| _ d| _dS )z�Verify that conditions are met so that this command can run. These include that there are enabled repositories with gpg keys, and that this command is being run by the root user. TN)�cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo�upgrade_minimal� all_security�skipped_grp_specs)�selfr r r r � configure0 s zUpgradeCommand.configurec C s� | j r dnd}| jj| j|| jd� | jjs<| jjs<| jjrzd}|| �� O }|| � � O }|| � � O }|| � � O }|r�d S n| j � � d S tj�td���d S )N�eqZgte)�cmp_type�allFz No packages marked for upgrade.)r r Z _populate_update_security_filterr r r � pkg_specs� grp_specs�_update_modules� _update_files�_update_packages�_update_groupsr Z upgrade_allr � exceptions�Errorr )r r �resultr r r �runB s � zUpgradeCommand.runc C sN t | jj�}tjjr6tjj�| j�}|� | jj�| _ n | jj| _ t | j �|kS )N) �lenr r# r r Z WITH_MODULES�module� module_baseZ ModuleBaser r )r Zgroup_specs_numr. r r r r$ V s z UpgradeCommand._update_modulesc C s� d}| j jr�| jj| j jd| jjjd�D ]`}z| j�|� d}W q( tjj y� } z*t � t d�| jjj �|j�� W Y d }~q(d }~0 0 q(|S )NF)�strict�progressT�No match for argument: %s)r r r Zadd_remote_rpms�outputr0 Zpackage_upgrader r( � MarkingError�logger�infor �term�bold�location)r �success�pkg�er r r r% ` s � � z UpgradeCommand._update_filesc C sp d}| j jD ]^}z| j�|� d}W q tjjyh } z(t�t d�| jj j � |�� W Y d }~q d }~0 0 q |S )NFTr1 ) r r" r r r r( r3 r4 r5 r r2 r6 r7 )r r9 Zpkg_specr; r r r r& m s � z UpgradeCommand._update_packagesc C s | j r| j�| j � dS dS )NTF)r r Zenv_group_upgrade)r r r r r' x s z UpgradeCommand._update_groupsN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r+ r$ r% r& r' r r r r r # s r )Z __future__r r ZloggingZdnf.exceptionsr Zdnf.baseZdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Z getLoggerr4 ZCommandr r r r r �<module> s __pycache__/install.cpython-39.opt-1.pyc 0000644 00000014005 15025176534 0014022 0 ustar 00 a �!cS � @ s� d dl mZ d dl mZ d dlZd dlmZ d dlZd dlZd dl m Z d dl m Z d dl mZ e�d�ZG d d � d e j�ZdS ) � )�absolute_import)�unicode_literalsN)�chain)�commands)� OptionParser)�_�dnfc @ s� e Zd ZdZejejejd�ZdZ de e� � � Z e d�Zedd� �Zdd � Zd d � Zd d � Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� ZdS )�InstallCommandzUA class containing methods needed by the cli to execute the install command. )z install-nz install-naz install-nevrazalternative-for({}))�install� localinstall�inz,install a package or packages on your systemc C s" | j ddtd�tjtd�d� d S )N�package�+ZPACKAGEzPackage to install)�nargs�metavar�action�help)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/install.py� set_argparser1 s �z InstallCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� dS )z�Verify that conditions are met so that this command can run. That there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. TN) �cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure7 s zInstallCommand.configurec C sL g }g }g }| � � }| j�| j� | jjdkrf| jjs>| jjrf| �| jj� | jj j rft j � td���g }| jj�rP| jjdk�rPt jj�rHz,t jj�| j�}|j| jj| jj j d� W n� t j j�yD } zn|jr�|jD ]}|�|� q�|j�r |jD ]}|�d| � q�|j} | �r0t�t jj�| d �� W Y d }~n d }~0 0 n| jj}| jj �r�|�r�| � | jj � | jj j �r�t j � td���n| � � }|�r�|�r�| � |� | jj j �r�t j � td���n |�r�| jjdk�r�| � |� | jjdk�r�| � |�}t!|�dk�s t!|�dk�s |�rH| jj j �rHt j j"td�d�#|�|d��d S ) Nr zNothing to do.��strict�@r zUnable to find a match� )�pkg_specZpackages)$� _get_nevra_forms_from_commandr Z _populate_update_security_filterr �command� grp_specs� pkg_specs� _log_not_valid_rpm_file_pathsr �confr r � exceptions�Errorr Z WITH_MODULES�module� module_baseZ ModuleBaser Z MarkingErrorsZno_match_group_specs�appendZerror_group_specs�module_depsolv_errors�logger�errorZ format_modular_solver_errorsr � _inform_not_a_valid_combination�_install_files�_install_groups�_install_packages�lenZPackagesNotAvailableError�join) r �err_pkgs�errsZerror_module_specs� nevra_formsZskipped_grp_specsr- �eZe_specr/ r r r �runE s\ � . �zInstallCommand.runc C s& | j j| jv r | j| j j gS g S d S )N)r r% r: )r r r r r$ z s z,InstallCommand._get_nevra_forms_from_commandc C sF t dd� |�}t| jj|�D ]$}td�}t�|| jjj � |�� q d S )Nc S s d| S )Nr! r )�gr r r �<lambda>� � z>InstallCommand._log_not_valid_rpm_file_paths.<locals>.<lambda>z Not a valid rpm file path: %s) �mapr r r' r r0 �infor �output�term�bold)r r& Z group_names�pkg�msgr r r r( � s z,InstallCommand._log_not_valid_rpm_file_pathsc C s. |D ]$}t d�}t�|| jjj�|�� qd S )NzNot a valid form: %s)r r0 Zwarningr rB rC rD )r �formsZformrF r r r r2 � s z.InstallCommand._inform_not_a_valid_combinationc C s� g }| j jj}| j j| jj|| j jjd�D ]\}z| j j||d� W q* t j j y� t d�}t �|| j jj�|j�� |�|� Y q*0 q*|S )N)r �progressr zNo match for argument: %s)r r) r Zadd_remote_rpmsr r rB rH Zpackage_installr r* � MarkingErrorr r0 rA rC rD �locationr. )r r8 r rE rF r r r r3 � s � z InstallCommand._install_filesc C sN z&| j j|t| j jj�| j jjd� W n" tjjyH | j jjrD� Y n0 d S )Nr ) r Zenv_group_install�tupler) Zgroup_package_typesr r r* r+ )r r& r r r r4 � s � z InstallCommand._install_groupsc C sV | j j�� j| j�|�d�}|rRtd�}t�|�|d� t t dd� |D ������ d S )N)Zprovidesz/There are following alternatives for "{0}": {1}z, c S s g | ] }|j �qS r )�name)�.0Zaltr r r � <listcomp>� r? z7InstallCommand._report_alternatives.<locals>.<listcomp>) r Zsack�queryZfilterm�alternatives_provide�formatr r0 rA r7 �sorted�set)r r# rO rF r r r �_report_alternatives� s ��z#InstallCommand._report_alternativesc C s� g }| j jj}| jjD ]�}z| j j|||d� W q tjjy� } zPd� |j | j j j � |��}t�|� | j �|� | �|� |�|� W Y d }~qd }~0 0 q|S )N)r rG z{}: {})r r) r r r' r r r* rI rQ �valuerB rC rD r0 rA Z_report_icase_hintrT r. )r r: r9 r r# r; rF r r r r5 � s "z InstallCommand._install_packagesN)�__name__� __module__� __qualname__�__doc__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr: rP rK �keys�aliasesr Zsummary� staticmethodr r r< r$ r( r2 r3 r4 rT r5 r r r r r % s& � 5 r )Z __future__r r Zlogging� itertoolsr rZ Zdnf.exceptionsr Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Z getLoggerr0 ZCommandr r r r r �<module> s __pycache__/swap.cpython-39.pyc 0000644 00000003633 15025176534 0012374 0 ustar 00 a �!cA � @ s` d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dl Z e � d�Z G dd� dej �Z dS ) � )�absolute_import)�unicode_literals)�_)�commandsN�dnfc @ sL e Zd ZdZdZed�jejj d�Z e dd� �Z dd� Z d d � Zd d � Zd S )� SwapCommandzNA class containing methods needed by the cli to execute the swap command. )Zswapz=run an interactive {prog} mod for remove and install one spec)�progc C s, | j ddtd�d� | j ddtd�d� d S )N� remove_spec�storez The specs that will be removed)�action�help� install_specz The specs that will be installed)� add_argumentr )�parser� r �9/usr/lib/python3.9/site-packages/dnf/cli/commands/swap.py� set_argparser& s �zSwapCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � t� | j| j j g� d S )NT) �cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�baseZ_checkEnabledRepo�optsr )�selfr r r r � configure, s zSwapCommand.configurec C s@ | j j�|�}|d ur<|| j �}| j j�|||g� |�� d S )N)r Z cli_commands�getZ optparserZparse_command_args�run)r Zcmd_str�specZcmd_cls�cmdr r r �_perform5 s zSwapCommand._performc C s$ | � d| jj� | � d| jj� d S )N�install�remove)r r r r )r r r r r < s zSwapCommand.runN)�__name__� __module__� __qualname__�__doc__�aliasesr �formatr �utilZMAIN_PROG_UPPERZsummary� staticmethodr r r r r r r r r s � r )Z __future__r r Zdnf.i18nr Zdnf.clir Zdnf.utilr ZloggingZ getLogger�loggerZCommandr r r r r �<module> s __pycache__/updateinfo.cpython-39.opt-1.pyc 0000644 00000033276 15025176534 0014525 0 ustar 00 a �!c:J � @ s� d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlm Z ddl m Z ddl m Z mZ dd lmZ d d � ZG d d � d e j�ZdS )zUpdateInfo CLI command.� )�absolute_import)�print_function)�unicode_literalsN)�commands)� OptionParser)�_� exact_width��unicodec C s t dd� | D ��S )z7Return maximum length of items in a non-empty iterable.c s s | ]}t |�V qd S �N)r )�.0�item� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/updateinfo.py� <genexpr>& � z_maxlen.<locals>.<genexpr>)�max)�iterabler r r �_maxlen$ s r c s0 e Zd ZdZejed�ejed�ejed�ej ed�ej ed�iZ ed�ed�ed �ed �d �Z d d d d d d dd�Z ddgee �� � Zed�ZdZdddegZ� fdd�Zedd� �Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Z d'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z!d1d2� Z"d3d4� Z#d5d6� Z$� Z%S )7�UpdateInfoCommandz)Implementation of the UpdateInfo command.�bugfix� enhancement�security�unknown� newpackagez Critical/Sec.zImportant/Sec.z Moderate/Sec.zLow/Sec.)�Critical� Important�Moderate�Low�list�info�summary)zlist-updateinfoz list-securityzlist-seczinfo-updateinfoz info-securityzinfo-seczsummary-updateinfoZ updateinfoZupifz!display advisories about packages� available� installed�updates�allc s t t| ��|� d| _dS )zInitialize the command.N)�superr �__init__�_installed_query)�self�cli�� __class__r r r' C s zUpdateInfoCommand.__init__c C s | � � }|jddddtd�d� |jddddtd �d� |jd dd dtd �d� |jd dddtd�d� g d�}| � � }|jddddtd�d� |jddddtd�d� |jddddtd�d� | jdd d d td �d � | jd!d"d d td#�d � | jd$d%d&||d' tjtd(�d)� d S )*Nz --available� _availabilityr" � store_constz?advisories about newer versions of installed packages (default))�dest�const�action�helpz --installedr# z?advisories about equal and older versions of installed packagesz --updatesr$ zbadvisories about newer versions of those installed packages for which a newer version is availablez--allr% z3advisories about any versions of installed packages)r! r r z --summary� _spec_actionr! z$show summary of advisories (default)z--listr zshow list of advisoriesz--infor zshow info of advisoriesz --with-cve�with_cveF� store_truez'show only advisories with CVE reference)r/ �defaultr1 r2 z --with-bz�with_bzz,show only advisories with bugzilla reference�spec�*ZSPECr zPackage specification)�nargs�metavar�choicesr6 r1 r2 )� add_mutually_exclusive_group� add_argumentr r ZPkgNarrowCallback)�parser� availabilityZcmdsZ output_formatr r r � set_argparserH sX ���� � � � � � �z UpdateInfoCommand.set_argparserc C s� d| j j_d| j j_| jj| jv r6| j| jj | j_n| jjrJ| jj| j_| jj r`| jj | j_ n8| jj rz| jj d | j vr�| j | j_ n| jj �d�| j_ t� | j_| jjr�| jj�tj� | jjr�| jj�tj� | jjr�| jj�tj� | jj�r| jj�tj� | jj �r�| jj �d�}|dk�r8| jj�tj� n�|dk�rT| jj�tj� np|dv �rp| jj�tj� nT|dk�r�| jj�tj� n8|dv �r�d| j_n$|dk�r�d| j_ n| jj � d|� | jj �r�| jj � | jj � d S ) zADo any command-specific configuration based on command arguments.Tr r r )r Zsecr )Z bugzillasZbzs�cvesN) r* ZdemandsZavailable_reposZsack_activation�optsZcommand�direct_commands� spec_actionr3 r- r@ r8 �availabilities�availability_default�pop�set�_advisory_typesr �add�hawkey�ADVISORY_BUGFIXr �ADVISORY_ENHANCEMENTr �ADVISORY_NEWPACKAGEr �ADVISORY_SECURITYr7 r4 �insert�advisory�extend)r) r8 r r r � configuren sJ zUpdateInfoCommand.configurec C s� | j jdkr$| �| j j�}td�}n^| j jdkrH| �| j j�}td�}n:| j jdkrl| �| j j�}td�}n| �| j j�}td�}| j jdkr�| � |� n$| j jdkr�| � |� n | � ||� dS )z#Execute the command with arguments.r# r$ r% r" r r N) rC r@ �installed_apkg_adv_instsr8 r �updating_apkg_adv_insts�all_apkg_adv_insts�available_apkg_adv_instsrE � display_list� display_info�display_summary)r) �apkg_adv_insts� descriptionr r r �run� s zUpdateInfoCommand.runc C s@ | j d u r | jj�� �� �� | _ | j j|j|jd�}t |�dkS )N)�nameZevr__gter ) r( �base�sack�queryr# Zapply�filterr_ �evr�len)r) �apackage�qr r r �_newer_equal_installed� s z(UpdateInfoCommand._newer_equal_installedc s | j js<| j js<| j js<| j js<| j js<| j js<| j js<dS � j| j jv rNdS t � fdd�| j jD ��rldS | j jr�� j| j jv r�dS | j jr�t � fdd�| j jD ��r�dS | j jr�t � fdd�| j jD ��r�dS | j jr�t dd� � j D ��r�dS | j j�rt dd� � j D ���rdS d S ) NTc 3 s | ]}t �� j|�V qd S r )�fnmatch� fnmatchcase�id�r �pat�rR r r r � r z6UpdateInfoCommand._advisory_matcher.<locals>.<genexpr>c s g | ]}� � |��qS r )Z match_bug)r Zbugrn r r � <listcomp>� r z7UpdateInfoCommand._advisory_matcher.<locals>.<listcomp>c s g | ]}� � |��qS r )Z match_cve)r Zcvern r r ro � r c S s g | ]}|j tjk�qS r )�typerL � REFERENCE_CVE�r �refr r r ro � r c S s g | ]}|j tjk�qS r )rp rL �REFERENCE_BUGZILLArr r r r ro � r F) rC rJ r8 �severityZbugzillarB r4 r7 rp �any� references)r) rR r rn r �_advisory_matcher� s> ������"" z#UpdateInfoCommand._advisory_matcherc # sd |� |�D ]T� � �| jj�}| �|�}t� fdd�| jjD ��}|sH|r | �� �}� ||fV q dS )z4Return (adv. package, advisory, installed) triplets.c 3 s | ]}t �� j|�V qd S r )ri rj r_ rl �rf r r r � s �zAUpdateInfoCommand._apackage_advisory_installed.<locals>.<genexpr>N) Zget_advisory_pkgsZ get_advisoryr` ra rx rv rC r8 rh )r) Z pkgs_queryZcmptype�specsrR Zadvisory_matchZapackage_matchr# r ry r � _apackage_advisory_installed� s � z.UpdateInfoCommand._apackage_advisory_installedc C s@ | j j}|�� jdd�}|�� }|r<|�|�� j|jd��}|S )z<Return query containing packages of currently running kernelT)�empty)� sourcerpm)r` ra rb �filtermZget_running_kernel�unionr} )r) ra rg Zkernelr r r �running_kernel_pkgs� s z%UpdateInfoCommand.running_kernel_pkgsc C s8 | j j�� �� �d�}|�| �� �� �}| �|tj |�S )z5Return available (adv. package, adv., inst.) triplets� ) r` ra rb r# Zlatestr r� r{ rL �GT)r) rz rg r r r rX � s z*UpdateInfoCommand.available_apkg_adv_instsc C s" | � | jj�� �� tjtjB |�S �z5Return installed (adv. package, adv., inst.) triplets)r{ r` ra rb r# rL �LT�EQ�r) rz r r r rU � s �z*UpdateInfoCommand.installed_apkg_adv_instsc C s | � | jj�� jdd�tj|�S )z4Return updating (adv. package, adv., inst.) tripletsT)Z upgradable)r{ r` ra rb r~ rL r� r� r r r rV � s �z)UpdateInfoCommand.updating_apkg_adv_instsc C s( | � | jj�� �� tjtjB tjB |�S r� ) r{ r` ra rb r# rL r� r� r� r� r r r rW s �z$UpdateInfoCommand.all_apkg_adv_instsc C sR i }|D ]:\}}}|j ||j< |j tjkr|j |jf||j|jf< qt�|�� �S )z Make the summary of advisories.)rp rk rL rP ru � collections�Counter�values)r) r\ �id2type�apkgrR r# r r r �_summary s zUpdateInfoCommand._summaryc C sR | � |�}|�r8ttd�| � dtd�|tj fdtd�|tj fdtd�|tjdf fdtd�|tjd f fdtd �|tjd f fdtd �|tjd f fdtd�|tjdf fdtd�|tj fdtd�|tj fdtd�|tj fg }t dd� |D ��}|D ]4\}}}|�s�qtd|d| t |�|f � �q| j j j �rN| j�� dS )z"Display the summary of advisories.z Updates Information Summary: r zNew Package notice(s)zSecurity notice(s)r� zCritical Security notice(s)r z Important Security notice(s)r zModerate Security notice(s)r zLow Security notice(s)r zUnknown Security notice(s)NzBugfix notice(s)zEnhancement notice(s)zother notice(s)c s s" | ]}|d rt |d �V qdS )� Nr )r �vr r r r ( r z4UpdateInfoCommand.display_summary.<locals>.<genexpr>z %*s %s� )r� �printr rL rO rP rM rN �ADVISORY_UNKNOWNr r r` �confZautocheck_running_kernelr* Z_check_running_kernel) r) r\ r] Ztyp2cntZ label_counts�width�indent�label�countr r r r[ s> � � � � ��" z!UpdateInfoCommand.display_summaryc s� � fdd�}� fdd�}t � }|D ]�\}}}d|j|j|jf }� jjsP� jjr�|jD ]V} | jt j krr� jjsrqVn| jt j kr�� jjs�qV|j|j f|� |||jft � �| j< qVq"|j|j f|� |||jft � �|j< q"g } d } } } t|�� dd� d �D ]l\\}}}}t| t|��} |�� D ]F\}}t| t|��} ||� }t| t|��} | �||�||||f� �q q�| D ]R\}}}}}� jjj�r�td || || || ||f � ntd || || ||f � �qld S ) z Display the list of advisories.c s � j jdksdS | rdS dS d S )Nr% � zi � )rC r@ )�inst�r) r r � inst2mark2 s z1UpdateInfoCommand.display_list.<locals>.inst2markc s2 | t jkr � j�|td��S � j�| td��S d S )Nz Unknown/Sec.r )rL rP �SECURITY2LABEL�getr � TYPE2LABEL)�typZsevr� r r � type2label: s z2UpdateInfoCommand.display_list.<locals>.type2labelz%s-%s.%sr c S s | d S )Nr r ��xr r r �<lambda>R r z0UpdateInfoCommand.display_list.<locals>.<lambda>��keyz%s%-*s %-*s %-*s %sz%s%-*s %-*s %sN)�dictr_ rd �archrC r4 r7 rw rp rL rt rq ru � setdefault�updatedrk �sorted�itemsr re �appendr` r� �verboser� )r) r\ r� r� Znevra_inst_dictr� rR r# Znevrars ZadvlistZidwZtlwZnwr� Zaupdatedr� ZaidZatypesevr� r r� r rY 0 s8 � � " z UpdateInfoCommand.display_listc s� �j j�� � �j jj�td�td�td�td�td�td�td�td�td �td �f �� ���fd d �}t� }|D ]\}}}|�|||�� qrtd � t |dd� d��� dS )z/Display the details about available advisories.z Update ID�TypeZUpdatedZBugsZCVEsZ DescriptionZSeverityZRightsZFilesZ Installedc s� | j g�j�| jtd��gt| j�gg g | jp0d�� | j g| j pBd�� t t � fdd�| j D ���d g }| jD ]L}|jtjkr�|d �d�|j |jp�d�� ql|jtjkrl|d �|j � ql|d �� |d �� �s�d |d< d |d < �jjd k�r|�rtd �ntd �g|d < t��}g }|�d� |�d| j � |�d� t�|�D ]f\}}|d d gfv �rf�qJt|�D ]>\}} |dk�r�|nd} |t| � } |�d| d| | f � �qn�qJd� |�S )Nr r� c 3 s | ]}|j � v r|jV qd S r )r� �filename)r �pkg)�archesr r r s s �zHUpdateInfoCommand.display_info.<locals>.advisory2info.<locals>.<genexpr>� z{} - {}r� � � r% �trueZfalse� zO===============================================================================r� r z %*s%s: %s� ) rk r� r� rp r r r� r] � splitlinesru Zrightsr� rI Zpackagesrw rL rt r� �format�titlerq �sortrC r@ r �zip� enumerater �join) rR r# Z attributesrs r� �linesr� Z atr_lines�i�liner� Z key_padding�r� �labelsr) r� r r � advisory2infoi sH � z5UpdateInfoCommand.display_info.<locals>.advisory2infoz c S s | � � S r )�lowerr� r r r r� � r z0UpdateInfoCommand.display_info.<locals>.<lambda>r� N) r` ra Z list_archesr� r� r rI rK r� r� r� )r) r\ r� Z advisoriesr� rR r# r r� r rZ a s �(z UpdateInfoCommand.display_info)&�__name__� __module__� __qualname__�__doc__rL rM r rN rP r� rO r� r� rD r �keys�aliasesr! rG rF r' � staticmethodrA rT r^ rh rx r{ r� rX rU rV rW r� r[ rY rZ � __classcell__r r r+ r r ) sR ��� %6 1r )r� Z __future__r r r r� ri rL Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr r Z dnf.pycompr r ZCommandr r r r r �<module> s __pycache__/autoremove.cpython-39.pyc 0000644 00000003664 15025176534 0013614 0 ustar 00 a �!c� � @ st d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z d dl Z e � d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_N�dnfc @ sR e Zd Zejejejd�Zdee� � � Z e d�Z e dd� �Zdd� Zdd � Zd S ) �AutoremoveCommand)z autoremove-nz autoremove-nazautoremove-nevra)� autoremovezKremove all unneeded packages that were originally installed as dependenciesc C s" | j ddtd�tjtd�d� d S )NZpackages�*zPackage to removeZPACKAGE)�nargs�help�action�metavar)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/autoremove.py� set_argparser, s �z AutoremoveCommand.set_argparserc C s\ | j j}d|_d|_d|_t| jj| jj| jj g�rLd| j j _ d|_ d|_n d|_d|_d S )NTF)Zcli�demandsZ resolvingZ root_userZsack_activation�any�opts� grp_specs� pkg_specs� filenames�baseZconfZ clean_requirements_on_removeZ allow_erasingZavailable_reposZfresh_metadata)�selfr r r r � configure2 s zAutoremoveCommand.configurec C sj t | jj| jj| jjg�r\g }| jj| jv r<| j| jj g}| j�|| jj| jj| jj� n | j�� d S )N) r r r r r Zcommand� nevra_formsr r )r Zformsr r r �runB s �zAutoremoveCommand.runN)�__name__� __module__� __qualname__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr �tuple�keys�aliasesr Zsummary� staticmethodr r r r r r r r " s � r )Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.exceptionsr r" ZloggingZ getLogger�loggerZCommandr r r r r �<module> s __pycache__/repoquery.cpython-39.pyc 0000644 00000053547 15025176534 0013466 0 ustar 00 a �B�f � � @ s d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ d dlm Z d dl Z d dl Z d dl Z d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZe �d�Zd Ze �d �Zd Zd d dddddddd� Zdd� ZG dd� de j �Z G dd� dej �Z G d d � d e �Z!dS ) � )�absolute_import)�print_function)�unicode_literals)�_)�commands)� OptionParserN�dnfz.%{name}-%{epoch}:%{version}-%{release}.%{arch}z%(-?\d*?){([:.\w]+?)}a name, arch, epoch, version, release, reponame (repoid), from_repo, evr, debug_name, source_name, source_debug_name, installtime, buildtime, size, downloadsize, installsize, provides, requires, obsoletes, conflicts, sourcerpm, description, summary, license, url, reason� conflicts�enhances� obsoletes�provides� recommends�requiresZ requires_pre�suggests� supplements) r r r r r r � requires-prer r c C s� dd� }dd� }| � dd�� dd�} t�� D ]\}}| � ||�} q,d }d }t�| �D ]0}||| ||�� � �7 }|||�7 }|�� }qT||| |d � �7 }|S ) z:Convert a rpm like QUERYFMT to an python .format() string.c S s^ | � � d }| � � d }|rJ|d dkr:d|dd � }nd| }d| }d|�� | d S ) Nr � �-�>�<�:z{0.�})�groups�lower)Zmatchobj�fill�key� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/repoquery.py�fmt_replD s z rpm2py_format.<locals>.fmt_replc S s | � dd�� dd�S )N�{z{{r z}})�replace)Ztxtr r r �bracketsO s z rpm2py_format.<locals>.bracketsz\n� z\t� � r N)r � OPTS_MAPPING�items� QFORMAT_MATCH�finditer�start�end)� queryformatr r! r �valueZfmt�spos�itemr r r � rpm2py_formatB s r/ c @ s e Zd ZdZdS )�_CommaSplitCallbackz\s*,\s*N)�__name__� __module__� __qualname__ZSPLITTERr r r r r0 _ s r0 c @ s� e Zd ZdZejejejd�Zde e� � � Z e d�Z edd� �Zedd� �Zd d � Zd d � Zd d� Zdd� Zd$dd�Zd%dd�Zd&dd�Zdd� Zdd � Zd d � Zd d � Zd'd"d#�Z dS )(�RepoQueryCommandzSA class containing methods needed by the cli to execute the repoquery command. )z repoquery-nz repoquery-nazrepoquery-nevra)Z repoqueryZrqz$search for packages matching keywordc C s, | j r|j| j d� | jr(|j| jd� |S )z'Filter query by repoid and arch options)Zreponame)�arch)Zrepo�filterm�arches)�opts�queryr r r �filter_repo_archm s z!RepoQueryCommand.filter_repo_archc C s� | j ddddtd�d� | j ddtd�d � | j d d d g td td�d� | j ddddtd�d� | j dg tdtd�d� | j dg tdtd�d� | j d g tdtd �d� | j d g tdtd �d� | j d g tdtd!�d� | j d"g tdtd#�d� | j d$g tdtd%�d� | j d&g tdtd'�d� | j d(g tdtd)�d� | �� }|j d*dtd+�d � |j d,dtd-�d � | j d.dtd/�d � | j d0dtd1�d � | j d2dtd3�d � | j d4dtd5�d � | j d6dtd7�d � | j d8d9ttd:�d;� | j d<dtd=�d � | �� }|j d>d?d@dAdtdB�dC� |j dDdEdFdAdtdG�dC� |j dHdIdJdAdtdK�dC� |j dLdMdAdtdN�dC� |j dOdPdQttdR�dS� | j dTdtdU�d � |j dVdQtdWtdX�dY� |j dZdQd[dWtd\�dY� |j d]dQd^dWtd_�dY� |j d`dtda�d � | �� }|j dbdcdddWtde�dY� |j dfdcdddWtjdY� |j dgdcdhdWtdi�dY� |j djdcdkdWtdl�dY� | j dmdtdn�d � | �� }tdo�tdp�tdq�tdr�tds�tdt�tdu�tdv�tdw�dx� }|�� D ]&\}}dy| }|j |dzdW||d{� �q�| j d|dtd}�d � td~�td�td��td��j t j j d��td��d��} | �� } | �� D ]&\} } dy| } | j | d�dW| | d{� �q.| j d�d�dWd�tjd{� | j d�dtd��d � | j d�d�d�td��d�� d S )�Nz-az--allZqueryall� store_truezNQuery all packages (shorthand for repoquery '*' or repoquery without argument))�dest�action�helpz--show-duplicatesz(Query all versions of packages (default))r= r> z--archz --archlistr7 z[arch]z show only results from this ARCH)r< �defaultr= �metavarr> z-fz--file�FILE�+z show only results that owns FILE)r@ �nargsr> z--whatconflictsZREQz#show only results that conflict REQ)r? r= r@ r> z --whatdependszjshows results that requires, suggests, supplements, enhances, or recommends package provides and files REQz--whatobsoletesz#show only results that obsolete REQz--whatprovidesz"show only results that provide REQz--whatrequiresz:shows results that requires package provides and files REQz--whatrecommendsz$show only results that recommend REQz--whatenhancesz"show only results that enhance REQz--whatsuggestsz"show only results that suggest REQz--whatsupplementsz%show only results that supplement REQ� --alldepsz=check non-explicit dependencies (files and Provides); default� --exactdepsz:check dependencies exactly as given, opposite of --alldeps� --recursivezOused with --whatrequires, and --requires --resolve, query packages recursively.z --deplistz>show a list of all dependencies and what packages provide themz --resolvez.resolve capabilities to originating package(s)z--treez"show recursive tree for package(s)z--srpmz#operate on corresponding source RPMz--latest-limit� latest_limitzOshow N latest packages for a given name.arch (or latest but N if N is negative))r< �typer> z--disable-modular-filteringz-list also packages of inactive module streamsz-iz--info� queryinfoFz+show detailed information about the package)r< r? r= r> z-lz--list� queryfilelistz!show list of files in the packagez-sz--source�querysourcerpmz show package source RPM namez --changelogs�querychangelogsz show changelogs of the packagez--qfz --queryformatr+ zfdisplay format for listing packages: "%%{name} %%{version} ...", use --querytags to view full tag list)r< r? r> z --querytagsz-show available tags to use with --queryformatz--nevra� store_constzZuse name-epoch:version-release.architecture format for displaying found packages (default))r< �constr= r> z--nvrz %{name}-%{version}-%{release}zQuse name-version-release format for displaying found packages (rpm query default)z--envraz.%{epoch}:%{name}-%{version}-%{release}.%{arch}zPuse epoch:name-version-release.architecture format for displaying found packagesz --groupmemberz=Display in which comps groups are presented selected packagesz --duplicates� pkgfilter� duplicatedz/limit the query to installed duplicate packagesz --duplicatedz --installonly� installonlyz1limit the query to installed installonly packagesz --unsatisfied� unsatisfiedzClimit the query to installed packages with unsatisfied dependenciesz --locationz5show a location from where packages can be downloadedz5Display capabilities that the package conflicts with.zaDisplay capabilities that the package can depend on, enhance, recommend, suggest, and supplement.z2Display capabilities that the package can enhance.z-Display capabilities provided by the package.z1Display capabilities that the package recommends.z1Display capabilities that the package depends on.z�If the package is not installed display capabilities that it depends on for running %%pre and %%post scriptlets. If the package is installed display capabilities that is depends for %%pre, %%post, %%preun and %%postun.z/Display capabilities that the package suggests.z5Display capabilities that the package can supplement.) r �dependsr r r r r r r z--%s� packageatr)r< r= rN r> � --availablez Display only available packages.z Display only installed packages.zLDisplay only packages that are not present in any of available repositories.zQDisplay only packages that provide an upgrade for some already installed package.zIDisplay only packages that can be removed by "{prog} autoremove" command.��progz2Display only packages that were installed by user.)� installed�extrasZupgrades�unneeded� userinstalled�listz --autoremoverZ z--recentz%Display only recently edited packagesr �*ZKEYzthe key to search for)rC r@ r> ) � add_argumentr r0 � add_mutually_exclusive_group�int�QFORMAT_DEFAULT�argparse�SUPPRESSr& �formatr �util� MAIN_PROG)�parserZwhatrequiresformZoutformrO Zpackage_attributeZ help_msgs�argZhelp_msg�nameZ help_listZ list_groupZlist_argZhelp_argZswitchr r r � set_argparserv s^ �� � � � � � � � � � � ��� � ���� �� � � �� �� � �� ������� � �� � � �z RepoQueryCommand.set_argparserc C s | j js | jjtjtjd� d S )N)�stdout�stderr)r8 �quiet�cliZredirect_logger�loggingZWARNING�INFO��selfr r r � pre_configure s z RepoQueryCommand.pre_configurec C sj | j js| j�� | jj}| j jrJ| j jrB| j�dd| j j � nd| j _| j jrVd S | j j rv| j jsvt j� t d���| j j r�| j jr�| j�dd� t| j j| j jdko�| j j g�s�t j� t d���| j js�| j j�r| j j�s| j j�st j� t d �| j j�rd nd����| j j�r$| jj�� | j jd v�r@| j jd k�sJ| j j�rPd |_d |_ | j j �rfd |_ d S )Nz --obsoletes�--r z�Option '--resolve' has to be used together with one of the '--conflicts', '--depends', '--enhances', '--provides', '--recommends', '--requires', '--requires-pre', '--suggests' or '--supplements' optionsrF rE r z�Option '--recursive' has to be used with '--whatrequires <REQ>' (optionally with '--alldeps', but not with '--exactdeps'), or with '--requires <REQ> --resolve'z;argument {} requires --whatrequires or --whatdepends optionrD )rX r[ rQ T) r8 rm rn Zredirect_repo_progress�demandsr rT Z_option_conflict� querytags�resolver ZCliErrorr � recursive� exactdeps�any� whatrequires�alldeps� whatdependsrd �srpm�baseZreposZenable_source_reposr\ rO � availableZavailable_reposZsack_activationrL � changelogs)rr ru r r r � configure sP ����� �� zRepoQueryCommand.configurec C s |j rlg }|�dt|� � |jD ]>}|d }|�d|�d�tj�|d �tj�|d �f � q"d�|�S zpt |�}|j r�| j j � |�W S |jr�|j}|s�ttd��|�tjd � |W S |jr�|jW S t|j��|�W S W n6 t�y } z tj�t|���W Y d }~n d }~0 0 d S ) NzChangelog for %s� timestampz * %s %s %s z %a %b %d %YZauthor�textr" z Package {} contains no files)�file) rL �append�strr� �strftimer �i18n�ucd�join�PackageWrapperrI r �outputZ infoOutputrJ �files�printr rd �sysrl rK Z sourcerpmr/ r+ �AttributeError� exceptions�Error) rr r8 �pkg�outZchlog�dtZpoZfilelist�er r r �build_format_fnG s0 � z RepoQueryCommand.build_format_fnc C sJ | j j�� jdd�}|D ],}|�|�tj�|�j | j jddd���}q|S )NT��emptyF)� with_provides�with_filenames) r �sackr9 r6 �union� intersectionr �subject�Subject�get_best_query)rr ZnevrasZ base_query�resolved_nevras_queryZnevrar r r �_resolve_nevrasc s ��z RepoQueryCommand._resolve_nevrasNc C sD |r|n|}|j |d�}|�|�}|�|�}|r@| j|||d�}|S )N�r ��done)�filter� differencer� �_do_recursive_deps)rr �query_in� query_selectr� Zquery_requiredr r r r� p s z#RepoQueryCommand._do_recursive_depsFc C s� | � ||�}|j|d�}|�|j|d��}|r�|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d��}|�|j|d ��}|�|j|d ��}| jjr�| �||�}|S ) N�Zrequires__globr� �Zrecommends__glob�Zenhances__glob�Zsupplements__glob�Zsuggests__glob�r �r �r �r )r� r� r� r8 rx r� )rr �namesr9 Z all_dep_typesr� Zdepqueryr r r � by_all_deps} s z RepoQueryCommand.by_all_depsc C s� |r|n| j j�� jdd�}| j j�� jdd�}|�� D ]}|�|j|jd��}q8|�|�}|rv| j |||�|�d�}|�|�S )NTr� �r r� ) r r� r9 r6 �runr� r� r r� � _get_recursive_providers_query)rr r� � providersr� �tr� r� r r r r� � s z/RepoQueryCommand._get_recursive_providers_queryc C st g }g }| j jD ]B}tjj�|�d }|�d�r<|�|� q|r|dv r|�|� q|rp| jj|d| jj j d�}|S )Nr z.rpm)�http�ftpr� �httpsF)�strict�progress) r8 r r Zpycomp�urlparse�endswithr� r Zadd_remote_rpmsr� r� )rr Zrpmnames�remote_packagesr Zschemesr r r �_add_add_remote_packages� s �z)RepoQueryCommand._add_add_remote_packagesc C s� | j jrtt� d S | j�| j � | jjj| j j r8t j nt j d�}| j j r�| �� }ddi}| j j| jv r�| j| j j g|d< d|d< g }|jdd�}|r�|�| jj�� j|d��}| j j D ]0}|�tjj|dd �j| jjfd |i|���}q�|}| j j�r |�| jjj�}| j j�r^| j j �r�| j j d k�r�t| jj � � � tj � t!d �"d d| j j ����nH| j j dk�r~|�#| jj$j%�}n(| j j �r�| j j dk�r�t&|| j j �� }| j j'dk�r�| j�(|�}|�)|��*� }n�| j j'dk�r�| j�(|�}n�| j j'dk�r\tj�+| j�}|�,| jjj-| jjj.� tj/�0|�} d| _1| j2dd�} | �sXttj3�4| �5� �� d S | j j �sn|�� }| �6| j |�}|} | j j7�r�|j| j j7d� | j j8�r�|j| j j8d�} | �|j| �9| j j8|�d��}| j j:�r�|j| j j:d� | j j;�r"|j| j j;d�} | �r| }n|j| j j;d� | j j<�rX| j j=�rH|j| j j<d� n| �>| j j<|�}| j j?�r�| j j=�r�|j| j j?d�}|�|j| j j?d��}|�|j| j j?d ��}|�|j| j j?d ��}|�|j| j j?d ��}n| �>| j j?|d�}| j j@�r |j| j j@d�} | �|j| �9| j j@|�d ��}| j jA�rX|j| j jAd �} | �|j| �9| j jA|�d ��}| j jB�r�|j| j jBd �} | �|j| �9| j jB|�d!��}| j jC�r�|j| j jCd �} | �|j| �9| j jC|�d"��}| j jD�r�|�E| j jD�}| jjF|dd#�}| j jG�rTg }|D ]<}|jH}|d u�r| jj�� j||jId$d%�}||�2� 7 }�q| jj�� j|d�}| j jJ�r�| j j<�s�| j jKd&v�r�tj � t!d'�j"tj3jLd(���| �M|| | j � d S tN� }| j jK�r�tN� } |�2� D ]p}| j j dk�s�| jj$�O|��r�| j jKd)k�r| �P|jQ|jR |jS |jT |jU � n| �Pt&|tV| j jK �� �q�| j jW�r�| j j d k�rf| �6| j | jj�� �}n| �6| j | jj�� �� �}|j| d*�}| j jX�r�|�| �Y||��}tN� }|�E� �2� D ]}|�Z| �[| j |�� �q�n|�Pd+d,� | D �� �n�| j j\�r*|�2� D ]"}|�]� }|d u�r|�Z|� �q�nf| j j^�r6g }t_tN|�2� ��D ]�}| j j dk�sj| jj$�O|��rHg }|�`d-ta|� � t_d.d/� |jQD ��D ]t}|�`d0| � tj�|�}|�| jj�}| �6| j |�� �}| j jb�s�|�E� }|�2� D ]}|�`d1ta|� � �q�q�|�`d2�c|�� �qH|�r2td3�c|�� d S | j jd�rN| �e|� d S |�2� D ]8}| j j dk�sx| jj$�O|��rV|�Z| �[| j |�� �qV|�r�| j jf�r�td3�ct_|��� ntd2�ct_|��� d S )4N)�flagsr� FZformsr� Tr� �r� )Z ignore_caser9 rX z)argument {}: not allowed with argument {}rU rt rZ r[ rP rQ rR )Zverify)Z file__glob)Zconflicts__glob)r )r )Zprovides__globr� r� r� r� r� r� r� r� r� )Zwarning�src)ri �evrr5 )r r r r r r r r z�No valid switch specified usage: {prog} repoquery [--conflicts|--enhances|--obsoletes|--provides|--recommends|--requires|--suggest|--supplements|--whatrequires] [key] [--tree] description: For the given packages print a tree of the packages.rV rS r� c s s | ]}t |�V qd S �N�r� )�.0�relr r r � <genexpr>R � z'RepoQueryCommand.run.<locals>.<genexpr>z package: c S s g | ] }t |��qS r r� )r� �reqr r r � <listcomp>^ r� z(RepoQueryCommand.run.<locals>.<listcomp>z dependency: z provider: r" z )gr8 rv r� � QUERY_TAGSrn Z _populate_update_security_filterr r� r9 Zdisable_modular_filtering�hawkeyZIGNORE_MODULAR_EXCLUDESZAPPLY_EXCLUDESr r� Zcommand� nevra_formsr� r� r6 r r� r� r� ZrecentZ_recentZconfr� r\ Z optparser� print_usager� r� r rd Z _unneeded�historyZswdb�getattrrO Z_get_installonly_queryr� rP Z rpmdb_sackZ _configureZinstallonlypkgsZinstallonly_limit�goalZGoalZprotect_running_kernelr� re Z_format_resolve_problemsZ problem_rulesr: r� Z whatconflictsr� Z whatobsoletesZ whatprovidesr{ ry r� r} ZwhatrecommendsZ whatenhancesZwhatsupplementsZ whatsuggestsrG ZlatestZ_merge_update_filtersr~ Z source_namer� ZtreerT rf � tree_seed�setZuser_installed�updater r r r r r% rw rx r� �addr� �locationZremote_locationZdeplist�sortedr� r� �verboser� Z groupmember�_group_member_reportrI )rr �qr� ZkwarkZpkgsZ query_resultsr rQ Zrpmdbr� ZsolvedZorqueryZrelsZquery_for_provideZ dependsqueryZpkg_listr� �srcnameZ tmp_queryr9 r� r� Zdeplist_outputr� r� Zproviderr r r r� � sj �� � ���� � � �� �� � � zRepoQueryCommand.runc C s i }| j jjD ] }tdd� |�� D ��||j< qi }g }|�� D ]b}g }|�� D ] \}} |j| v rP|� |� qP|r�|� d� t |��g �� t |�� q@|� t |�� q@g } t |�� �D ]4\} } | � d� t | �t dd� | �d�D �� �� q�| � d� t |��� | �rtd� | �� d S )Nc S s g | ] }|j �qS r �ri )r� r� r r r r� ~ r� z9RepoQueryCommand._group_member_report.<locals>.<listcomp>�$r" c S s g | ] }d | �qS )z @r )r� �idr r r r� � r� )r �compsr r� Z packages_iterr� r� r& ri r� � setdefaultr� r� r� �splitr� ) rr r9 Zpackage_conf_dict�groupZgroup_package_dictZpkg_not_in_groupr� Z group_id_listZgroup_idZpackage_name_setr� r Z package_listr r r r� { s2 ��$�z%RepoQueryCommand._group_member_reportc C s� | � ||�}|dkr t|� d S d}td|�D ] }|d7 }q.g }|jD ]}|�t|�� qFdtt|�� d d�|� d } t|d | d | � d S ) N���� r z| �[z: z, �]z\_ )r� r� �ranger r� r� �lenr� ) rr �levelr� r8 Z pkg_stringZspacing�xr Z requirepkgZreqstrr r r � grow_tree� s "zRepoQueryCommand.grow_treer� c C s, t t|�� �dd� d�D �] }|d u s.|dkr4t� n|}|j�d�sP|j�d�rV d S | �|||� ||vr|�|� |jr�t||j�}i }t|�D ]6} | j j � � j | d�} | D ]} | || jd | j < q�q�| j j � � j t|�� �d �} n&|j�r| �|jf|�n |j|jd �} | �| |||d |� qd S ) Nc S s | j S r� r� )�pr r r �<lambda>� r� z,RepoQueryCommand.tree_seed.<locals>.<lambda>)r r� ZrpmlibZsolvabler� �.r� r� r )r� r� r� ri � startswithr� r� rT r� r r� r9 r6 r5 r\ �valuesr| r� r� r� ) rr r9 Zaqueryr8 r� Zusedpkgsr� Zstrpkg�arri ZpkgqueryZquerypkgr r r r� � s& �zRepoQueryCommand.tree_seed)N)F)N)r� N) r1 r2 r3 �__doc__r� Z FORM_NAMEZFORM_NAZ FORM_NEVRAr� �tuple�keys�aliasesr Zsummary� staticmethodr: rj rs r� r� r� r� r� r� r� r� r� r� r� r r r r r4 c s2 � 0 Ir4 c @ sD e Zd ZdZdd� Zdd� Zedd� �Zedd � �Z ed d � �Z d S ) r� z>Wrapper for dnf.package.Package, so we can control formatting.c C s || _ d S r� )�_pkg)rr r� r r r �__init__� s zPackageWrapper.__init__c C sF t | j|�}|d u rdS t|t�r:d�tdd� |D ���S tj�|�S )Nz(none)r" c S s h | ]}t j�|��qS r )r r� r� )r� Zreldepr r r � <setcomp>� r� z-PackageWrapper.__getattr__.<locals>.<setcomp>) r� r� � isinstancer\ r� r� r r� r� )rr �attrZatrr r r � __getattr__� s zPackageWrapper.__getattr__c C s& | dkr t j �| �}|�d�S dS d S )Nr z%Y-%m-%d %H:%Mr$ )�datetimeZutcfromtimestampr� )r� r� r r r �_get_timestamp� s z PackageWrapper._get_timestampc C s | � | jj�S r� )r r� � buildtimerq r r r r � s zPackageWrapper.buildtimec C s | � | jj�S r� )r r� � installtimerq r r r r � s zPackageWrapper.installtimeN) r1 r2 r3 r� r� r r� r �propertyr r r r r r r� � s r� )"Z __future__r r r Zdnf.i18nr Zdnf.clir Zdnf.cli.option_parserr rb r ro �rer� r Zdnf.exceptionsZ dnf.subjectZdnf.utilr� Z getLogger�loggerra �compiler' r� r% r/ Z_SplitCallbackr0 ZCommandr4 �objectr� r r r r �<module> sL � X __pycache__/distrosync.cpython-39.pyc 0000644 00000002705 15025176534 0013622 0 ustar 00 a �!c� � @ s: d dl mZ d dlmZ d dlmZ G dd� dej�ZdS )� )�absolute_import)�commands)�_c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �DistroSyncCommandzZA class containing methods needed by the cli to execute the distro-synch command. )z distro-syncZ distrosyncz distribution-synchronizationZdsyncz?synchronize installed packages to the latest available versionsc C s | j ddtd�d� d S )N�package�*zPackage to synchronize)�nargs�help)� add_argumentr )�parser� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/distrosync.py� set_argparser" s z DistroSyncCommand.set_argparserc C sF | j j}d|_d|_d|_d|_t�| j| j � t� | j| j j � d S )NT) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�baseZ_checkEnabledRepo�optsr )�selfr r r r � configure& s zDistroSyncCommand.configurec C s | j �| jj�S )N)r Zdistro_sync_userlistr r )r r r r �run/ s zDistroSyncCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r r r r r r s r N)Z __future__r Zdnf.clir Zdnf.i18nr ZCommandr r r r r �<module> s __pycache__/downgrade.cpython-39.opt-1.pyc 0000644 00000003502 15025176534 0014326 0 ustar 00 a �!c � @ sR d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ G dd� dej �Z dS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �DowngradeCommandzWA class containing methods needed by the cli to execute the downgrade command. )Z downgradeZdgzDowngrade a packagec C s | j ddtd�tjd� d S )N�package�*zPackage to downgrade)�nargs�help�action)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/downgrade.py� set_argparser$ s �z DowngradeCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� d S )NT) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure) s zDowngradeCommand.configurec C sJ | j j| jjd| j jjd�}| j j| jjdd� | jjD � || j j j d�S )NF)�strict�progressc S s g | ] }d | �qS )�@r )�.0�xr r r � <listcomp>8 � z(DowngradeCommand.run.<locals>.<listcomp>)Zspecs� file_pkgsr ) r Zadd_remote_rpmsr r �outputr Z downgradePkgsZ pkg_specsZ grp_specsZconfr )r r r r r �run4 s ��zDowngradeCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r! r r r r r s r N) Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr ZCommandr r r r r �<module> s __pycache__/check.cpython-39.pyc 0000644 00000007261 15025176534 0012500 0 ustar 00 a �!c? � @ sV d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dlZ G dd� dej �Z dS )� )�absolute_import)�unicode_literals)�_)�commandsNc @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) � CheckCommandzSA class containing methods needed by the cli to execute the check command. )�checkz#check for problems in the packagedbc C s� | j ddddtd�d� | j ddddtd �d� | j d ddd td �d� | j d dddtd�d� | j ddddtd�d� | j ddddd ddg gtjd� d S )Nz--all� check_types� append_const�allzshow all problems; default)�dest�action�const�helpz--dependencies� dependencieszshow dependency problemsz --duplicates� duplicateszshow duplicate problemsz --obsoleted� obsoletedzshow obsoleted packagesz --provides�provideszshow problems with provides�check_yum_types�*)�nargs�choicesr )� add_argumentr �argparse�SUPPRESS)�parser� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/check.py� set_argparser$ s2 ����� ��zCheckCommand.set_argparserc C sx d| j j_| jjr<| jjr0| jj| jj | j_n | jj| j_| jjsPdh| j_nt| jj�| j_| jj j dg7 _ d S )NTr ) ZcliZdemandsZsack_activation�optsr r �set�base�confZdisable_excludes)�selfr r r � configure; s � zCheckCommand.configurec C sj t � }| jj�� �� }| jj�ddh��r�d }|D �]f}t |j�t t |j �t |j � �B D ]�}t |�� d�rnqZt |j|gd��sZt |�� d�r�|d u r�tj�| j�}tj�|�}|j t |�d� tj�|�}| jjj|_|j|dd� |�� }|r�qZtd�} |�| �| jj j � |�| jj j � |��� qZ|j D ]j} |j| gt | �� � d d �} | D ]B} d } |�| �| jj j � |�| jj j � | �| jj j � | ��� �qR�q.q2| jj�dd h��r4| j�!|�} |�"� �#| ��$� }|�%� D ]\\}}|�&� |d d � D ]<}td��| jj j � |d �| jj j � |��} |�| � �q�q�| jj�ddh��r�|D ]p}|j'D ]b}|j|gt |�� � d d �}t |��rVtd��| jj j � |d �| jj j � |��} |�| � �qV�qL| jj�ddh��r2|D ]Z}|j(D ]L}||j|gd�v�r�td�} |�| �| jj j � |�| jj j � |��� �q�q�t)|�D ]} t*| � �q:|�rftj+�,d�t |����d S )Nr r Zrpmlib)r �(F)ZselectZoptionalz {} has missing requires of {}r )r �namez"{} has installed conflict "{}": {}r � z{} is a duplicate with {}r z{} is obsoleted by {}r z%{} provides {} but it cannot be foundz Check discovered {} problem(s))-r r �sack�queryZ installedr r � intersectionZregular_requiresZ requires_preZprereq_ignoreinst�str� startswith�len�filter�dnfZ rpmdb_sack�selectorZSelector�goalZGoalr! Zprotect_running_kernel�install�runr �add�format�outputZtermZbold� conflicts�splitZ_get_installonly_queryZ duplicated� differenceZ _name_dict�items�sortZ obsoletesr �sorted�print� exceptions�Error)r" Z output_set�qr' �pkgZrequirer/ r0 Zsolved�msgZconflictZ conflictedZ conflict_pkgZ installonlyZdupsr% Zpkgs�dupZobsoleter Zprovider r r r2 I s� & � �� � � � � �zCheckCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r# r2 r r r r r s r ) Z __future__r r Zdnf.i18nr Zdnf.clir r Zdnf.exceptionsr. ZCommandr r r r r �<module> s __pycache__/search.cpython-39.opt-1.pyc 0000644 00000010551 15025176534 0013623 0 ustar 00 a �!c� � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlmZ d dlmZ d dl m Z m Z m Z d dl Z d dlZ d dlZ d dlZd dlZe�d�ZG d d � d ej�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�commands)� OptionParser)�ucd�_�C_�dnfc @ sP e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dS )� SearchCommandzTA class containing methods needed by the cli to execute the search command. )�searchZsez+search package details for the given stringc C s< | j ddtd�d� | j ddtd�dgd tjtd �d � d S ) Nz--all� store_truez'search also package description and URL)�action�help� query_string�+ZKEYWORD�allzKeyword to search for)�nargs�metavar�choices�defaultr r )� add_argumentr r ZPkgNarrowCallback)�parser� r �;/usr/lib/python3.9/site-packages/dnf/cli/commands/search.py� set_argparser0 s ��zSearchCommand.set_argparserc s$ t �dtdd�fdtdd�fdtdd�fdtd �ff�� � fd d ����fd d �}tj�� }|D ] }��|d|� ��|d|� q`�jj r�|D ] }��|d|� ��|d|� q�n6t |�}t |� � �}|D ] }t |� |��|kr�||= q�d}d} d} d} d} �jjj�s$�jj�� j|� � d��� } t� } |jd| d�D ]�}�jjj�sp|j|j | v �r^�q8| �|j|j � ||�|�k�r�|�|�}d} | |� |�k�r�|� |�} d} | |�|�| kk�r�|�|�| k} d} | �r�|| || � d} �jj � ||�|�|� �q8t |�dk�r t � td�� dS )z0Search for simple text tags in a package object.�nameZlong�Name�summaryZSummary� descriptionZ Description�urlZURLc s z � | W S | Y S 0 d S )Nr )�attr)� TRANS_TBLr r �_translate_attrC s z.SearchCommand._search.<locals>._translate_attrc s^ t � |�}td��|�}| r*td�| }n td�| }�jj�|d�|� �}tt|�� d S )Nz & z%s Exactly Matched: %%sz%s Matched: %%sz, )�mapr �join�base�outputZ fmtSection�printr )� exact_match�attrs�keysZ trans_attrsZtrans_attrs_strZ section_text� formatted)r# �selfr r �_print_section_headerI s z4SearchCommand._search.<locals>._print_section_headerNF)�pkgT)�reverseZlimit_tor zNo matches found.) � collections� OrderedDictr r r Z match_counterZ MatchCounter�_search_counted�optsr �len�listr+ �matched_needlesr& ZconfZshowdupesfromrepos�sack�query�filtermZlatest�set�sortedr Zarch�addZ matched_keysZmatched_haystacksr' Z matchcallback�logger�info)r- �argsr. �counter�argZneedlesZpkgsr/ Z used_attrsr7 r) Zprint_section_header�limit�seenr )r" r# r- r �_search9 sb � zSearchCommand._searchc C s` d| |i}t j�|�r$d| |i}| jj�� jtjfi |��}|� � D ]}|� |||� qH|S )Nz %s__substrz%s__glob) r �utilZis_glob_patternr& r8 r9 r: �hawkeyZICASE�runr= )r- rA r! ZneedleZfdict�qr/ r r r r3 � s z SearchCommand._search_countedc C s | j js | jjtjtjd� d S )N)�stdout�stderr)r4 �quiet�cliZredirect_logger�loggingZWARNING�INFO�r- r r r � pre_configure� s zSearchCommand.pre_configurec C sD | j js| j�� | jj}d|_d|_d|_| j jp:| j j | j _d S )NTF) r4 rL rM Zredirect_repo_progress�demandsZavailable_reposZfresh_metadataZsack_activationr Zquery_string_action)r- rR r r r � configure� s zSearchCommand.configurec C s t �td�� | �| jj�S )NzSearching Packages: )r> �debugr rE r4 r rP r r r rH � s zSearchCommand.runN)�__name__� __module__� __qualname__�__doc__�aliasesr r � staticmethodr rE r3 rQ rS rH r r r r r ( s O r )Z __future__r r r r1 Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr r r r Zdnf.match_counterZdnf.utilrG rN Z getLoggerr> ZCommandr r r r r �<module> s __pycache__/makecache.cpython-39.pyc 0000644 00000002434 15025176534 0013321 0 ustar 00 a �!cm � @ sx d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dlZd dl Zd dl Zd dl Z e � d�Z G dd� dej�ZdS ) � )�absolute_import)�unicode_literals)�commands)�_N�dnfc @ s, e Zd ZdZed�Zedd� �Zdd� ZdS )�MakeCacheCommand)Z makecacheZmczgenerate the metadata cachec C s, | j dddd� | j dddgdtjd� d S )Nz--timer� store_true� timer_opt)�action�dest�timer�?)�nargs�choices�metavar�help)� add_argument�argparse�SUPPRESS)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/makecache.py� set_argparser' s �z MakeCacheCommand.set_argparserc C s2 | j jd up| j j}td�}t�|� | j�|�S )Nz*Making cache files for all metadata files.)Zoptsr r r �logger�debug�baseZ update_cache)�selfr �msgr r r �run. s zMakeCacheCommand.runN) �__name__� __module__� __qualname__�aliasesr Zsummary� staticmethodr r r r r r r # s r )Z __future__r r Zdnf.clir Zdnf.i18nr r r Zdnf.exceptionsZdnf.utilZloggingZ getLoggerr ZCommandr r r r r �<module> s __pycache__/shell.cpython-39.opt-1.pyc 0000644 00000017400 15025176534 0013465 0 ustar 00 a �!cl& � @ s� d dl mZ d dlmZmZ d dlZd dlZd dlZd dlZd dl Z d dl Z d dl Z e � d�Z G dd� de�ZG dd� dejej�ZdS ) � )�commands)�_�ucdN�dnfc @ s e Zd ZdZdZdZdZdS )�ShellDemandSheetTN)�__name__� __module__� __qualname__Zavailable_reposZ resolvingZ root_userZsack_activation� r r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/shell.pyr % s r c @ s� e Zd ZdZed�jejjd�Z ddddddddd d d � Z d d � Z e dd� �Z dd� Zdd� Zdd� Zdd� Zd)dd�Zd*dd �Zd+d d �Zd,d d �Zd!d"� Zd-d#d$�Zd.d%d&�Zd/d'd(�ZdS )0� ShellCommand)�shell�shz run an interactive {prog} shell��prog�repo�quitZts_run� transaction�config�resolve�help) r � repository�exitr �run�tsr r � resolvedepr c C s$ t j�| |� tj�| � d| _d S )Nz> )r �Command�__init__�cmd�Cmd�prompt)�self�clir r r r = s zShellCommand.__init__c C s* | j ddtd�td�jtjjd�d� d S )N�script�?ZSCRIPTz Script to run in {prog} shellr )�nargs�metavarr )� add_argumentr �formatr �util�MAIN_PROG_UPPER)�parserr r r � set_argparserB s ��zShellCommand.set_argparserc C sl | j j}t� | j _t|�D ]L}|�d�r*qzt| j j|� W q tyd t| j j|t||�� Y q0 qd S )N�__)r" �demandsr �dir� startswith�getattr�AttributeError�setattr)r! Zdefault_demands�attrr r r � configureH s zShellCommand.configurec C s$ | j jr| �| j j� n| �� d S �N)�optsr# � _run_scriptZcmdloop�r! r r r r U s zShellCommand.runc C s | j �� d | j _| j �� d S r6 )�baseZ_finalize_base� _transaction� fill_sackr9 r r r �_clean[ s zShellCommand._cleanc C sh |r |dkrd S |dkr d}zt �|�}W n | �� Y d S 0 | jjjdd� | jj�|�}|jdkrnd S |j| jv r�t | d| j|j �|dd � � n�| jj � |j�}|d u�r\|| j�}z| jj� ||�}W n t y� Y d S 0 z&t�| jj�|j_|�� |�� W nF tjj�yX } z(t�td �d t|� � W Y d }~d S d }~0 0 n| �� d S ) N� ZEOFr F)Z reset_usager r � �Error:� )�shlex�split�_helpr" � optparserr Zparse_main_argsZcommand�MAPPINGr1 Z cli_commands�getZparse_command_args� SystemExit�copy�deepcopyr. r5 r r � exceptions�Error�logger�errorr r )r! �lineZs_liner7 Zcmd_clsr �er r r �onecmd` s< $ zShellCommand.onecmdNc C s� dd� }|rt |�dkr&| �d� d S |d }t |�dkrB|d nd }|�d�}|dkr�|d |� }||d d � }| jj�|�}|D ]}||||� q�|s�t�td �|� n|||| jj � d S ) Nc S sP |rt || |� n:z td�| t|t| ���� W n t�td�� Y n0 d S )Nz{}: {}zUnsupported key value.)r3 �printr( r1 �strrM �warningr )�key�val�confr r r � print_or_set� s z*ShellCommand._config.<locals>.print_or_set� r r r? �.���z Could not find repository: %s) �lenrD �findr: �repos� get_matchingrM rT r rW ) r! �argsrX rU rV ZperiodZ repo_namer^ r r r r �_config� s$ �zShellCommand._configc C s� t |t�r t|�dkr |d n|}d}|r�|dkrBtd��|�}n�|dkrZtd��|�}nv|dv rrtd��|�}n^|d kr�td ��|�}nF|d v r�td ��|�}n.|d kr�td��|�}n|dv r�td��|�}|s�| jj�� td�}td| � dS )z�Output help information. :param args: the command to output help information about. If *args* is an empty, general help will be output. r Nr z�{} arg [value] arg: debuglevel, errorlevel, obsoletes, gpgcheck, assumeyes, exclude, repo_id.gpgcheck, repo_id.exclude If no value is given it prints the current value. If value is given it sets that value.r z{} [command] print help)r r z�{} arg [option] list: lists repositories and their status. option = [all | id | glob] enable: enable repositories. option = repository id disable: disable repositories. option = repository idr z"{} resolve the transaction set)r r zy{} arg list: lists the contents of the transaction reset: reset (zero-out) the transaction run: run the transactionr z{} run the transaction)r r z{} exit the shella� Shell specific arguments: config set config options help print help repository (or repo) enable, disable or list repositories resolvedep resolve the transaction set transaction (or ts) list, reset or run the transaction set run resolve and run the transaction set exit (or quit) exit the shellr> ) � isinstance�listr\ r r( r" rE � print_helprR )r! r` �arg�msgr r r rD � sF "������� zShellCommand._helpc C s� |r |d nd }|dv r6| � dd�|dd � � � n�|dv r�| jjj}d}|dd � D ]L}|�|�}|r|t||�� d}qXt�t d �d t d � | jj j � |�� qX|r�| j� � d | j_n | �d � d S ) Nr �rc Nz repolist rA r? )�enable�disableFTr@ zUnknown repo: '%s'r )rQ �joinr" r: r^ r_ r1 rM Zcriticalr �output�term�boldr< Z_compsrD )r! r` r r^ r< r �rr r r �_repo� s$ � zShellCommand._repoc C sN z| j j�| j jj� W n0 tjjyH } zt|� W Y d }~n d }~0 0 d S r6 ) r" r: r r. Z allow_erasingr rK Z DepsolveErrorrR )r! r` rP r r r �_resolve� s zShellCommand._resolvec C s� zTt |d��6}|�� }|D ]}|�d�s| �|� qW d � n1 sH0 Y W n8 ty� t�td�| jj j � |�� t � d� Y n0 d S )Nrn �#z!Error: Cannot open %s for readingr? )�open� readlinesr0 rQ �IOErrorrM �infor r: rk rl rm �sysr )r! �file�fd�linesrO r r r r8 � s . zShellCommand._run_scriptc C s� |r |d nd }|dkr$| � � d S | �� |dv rZ| jjr�| jj�| jj�}t�|� n||dkr�z| j�� W nB t j j y� } z&t� t d�d t|� � W Y d }~nd }~0 0 t�t d�� | � � n | �d� d S ) Nr �resetrg r r@ rA z Complete!r )r= rp r: r; rk Zlist_transactionrM ru Zdo_transactionr rK rL rN r r rD )r! r` r �outrP r r r r; s" 0 zShellCommand._transactionc C s | � dg� d S )Nr )r; �r! r` r r r �_ts_run" s zShellCommand._ts_runc C s t �td�� t�d� d S )Nz Leaving Shellr )rM ru r rv r r| r r r �_quit% s zShellCommand._quit)N)N)N)N)N)N)N)r r r �aliasesr r( r r) r* ZsummaryrF r � staticmethodr, r5 r r= rQ ra rD ro rp r8 r; r} r~ r r r r r , s8 � & ; r )Zdnf.clir Zdnf.i18nr r Zdnf.utilr r rI ZloggingrB rv Z getLoggerrM �objectr r r r r r r r �<module> s __pycache__/module.cpython-39.pyc 0000644 00000034203 15025176534 0012704 0 ustar 00 a �!c�A � @ s� d dl mZ d dlmZmZ d dlmZ d dlmZ d dl m Z d dl Z d dl Z d dl Z d dlZd dlZd dlZ d dlZ G dd� dej�ZdS ) � )�print_function)�commands�CliError)�_)�NoModuleException)�loggerNc s* e Zd ZG dd� dej�ZG dd� de�ZG dd� de�ZG dd� de�ZG d d � d e�Z G d d � d e�Z G d d� de�Z G dd� de�Z G dd� de�Z G dd� de�ZG dd� de�ZG dd� de�Zeeee e e e e eeeh ZehZdZed�Z� fdd �Zd d � Zd d � Zd!d"� Zd#d$� Z� ZS )%� ModuleCommandc s, e Zd Z� fdd�Zdd� Zdd� Z� ZS )zModuleCommand.SubCommandc s( t tj| ��|� tjj�| j�| _d S �N) �superr � SubCommand�__init__�dnf�module� module_baseZ ModuleBase�base)�self�cli�� __class__� �;/usr/lib/python3.9/site-packages/dnf/cli/commands/module.pyr ( s z!ModuleCommand.SubCommand.__init__c C s� t � }| jjD ]�}| j�|�\}}|d u r,q|jr8|jnd}|jrH|jnd}|jr\|jdksb|jrvt � t d�� |�� |j r�|j nd}| jj�||dd|�}|�|� q|S )N� ���zjOnly module name, stream, architecture or profile is used. Ignoring unneeded information in argument: '{}')�set�opts� module_specr � _get_modules�name�stream�version�contextr �infor �format�archr �_moduleContainer�query�update) r �modules_from_specsr �__Znsvcapr r r# �modulesr r r �#_get_modules_from_name_stream_specs, s � z<ModuleCommand.SubCommand._get_modules_from_name_stream_specsc C s~ t � }t � }|D ](}||vr| jj�|�r|�|�� � q|D ]6}t�|�}|jtj gd�D ]}|j r\|� |j � q\q>||fS )N)Zforms) r r r$ ZisModuleActiver& Z getArtifacts�hawkeyZSubjectZget_nevra_possibilitiesZ FORM_NEVRAr �add) r Z use_modulesZ skip_modulesZ artifactsZ pkg_namesr ZartifactZsubjZ nevra_objr r r �_get_module_artifact_names> s � z3ModuleCommand.SubCommand._get_module_artifact_names)�__name__� __module__� __qualname__r r* r- � __classcell__r r r r r & s r c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ListSubCommand)�listz,list all module streams, profiles and statesc C s | j j}d|_d|_d S �NT�r �demands�available_repos�sack_activation�r r5 r r r � configureR s z&ModuleCommand.ListSubCommand.configurec C s� | j }| jjr&|�| jjtjjj�}nV| jj rF|�| jjtjjj �}n6| jj rf|�| jjtjjj �}n|�| jjtjjj �}|r�t|� d S | jjr�td�}tj�|��d S �NzNo matching Modules to list)r r �enabledZ_get_brief_descriptionr �libdnfr �ModulePackageContainerZModuleState_ENABLED�disabledZModuleState_DISABLED� installedZModuleState_INSTALLEDZModuleState_UNKNOWN�printr r � exceptions�Error)r Zmods�output�msgr r r � run_on_moduleW s0 ����z*ModuleCommand.ListSubCommand.run_on_moduleN�r. r/ r0 �aliasesr �summaryr9 rE r r r r �ListSubCommandM s rI c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.InfoSubCommand)r! z)print detailed information about a modulec C s | j j}d|_d|_d S r3 r4 r8 r r r r9 t s z&ModuleCommand.InfoSubCommand.configurec C sf | j jr| j�| j j�}n*| j jr4| j�| j j�}n| j�| j j�}|rRt|� nt j � t d���d S r: ) r �verboser Z_get_full_infor �profileZ_get_info_profilesZ _get_infor@ r rA rB r �r rC r r r rE y s z*ModuleCommand.InfoSubCommand.run_on_moduleNrF r r r r �InfoSubCommando s rM c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.EnableSubCommand)�enablezenable a module streamc C s$ | j j}d|_d|_d|_d|_d S r3 �r r5 r6 r7 � resolving� root_userr8 r r r r9 � s z(ModuleCommand.EnableSubCommand.configurec C s� z| j �| jj� W nn tjjy� } zR| jjj r`|j s>|j rB|�|j r`|j d t jjjkr`|�t�t|�� W Y d }~n d }~0 0 d S �N� )r rN r r r rA � MarkingErrorsr �conf�strict�no_match_group_specs�error_group_specs�module_depsolv_errorsr< r r= �!ModuleErrorType_ERROR_IN_DEFAULTSr �error�str�r �er r r rE � s �z,ModuleCommand.EnableSubCommand.run_on_moduleNrF r r r r �EnableSubCommand� s r_ c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.DisableSubCommand)�disablez%disable a module with all its streamsc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z)ModuleCommand.DisableSubCommand.configurec C s� z| j �| jj� W nn tjjy� } zR| jjj r`|j s>|j rB|�|j r`|j d t jjjkr`|�t�t|�� W Y d }~n d }~0 0 d S rR )r r` r r r rA rT r rU rV rW rX rY r< r r= rZ r r[ r\ r] r r r rE � s �z-ModuleCommand.DisableSubCommand.run_on_moduleNrF r r r r �DisableSubCommand� s ra c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ResetSubCommand)�resetzreset a modulec C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z'ModuleCommand.ResetSubCommand.configurec C sd z| j �| jj� W nJ tjjy^ } z.| jjj r<|j r<|�t � t |�� W Y d }~n d }~0 0 d S r )r rb r r r rA rT r rU rV rW r r[ r\ r] r r r rE � s z+ModuleCommand.ResetSubCommand.run_on_moduleNrF r r r r �ResetSubCommand� s rc c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.InstallSubCommand)�installz/install a module profile including its packagesc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z)ModuleCommand.InstallSubCommand.configurec C sr z | j �| jj| jjj� W nP tjj yl } z4| jjjrJ|j sF|j rJ|�t � t|�� W Y d }~n d }~0 0 d S r )r rd r r r rU rV r rA rT rW rX r r[ r\ r] r r r rE � s z-ModuleCommand.InstallSubCommand.run_on_moduleNrF r r r r �InstallSubCommand� s re c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.UpdateSubCommand)r& z0update packages associated with an active streamc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z(ModuleCommand.UpdateSubCommand.configurec C s& | j �| jj�}|r"td�|���d S )Nz, )r Zupgrader r r �join)r Z module_specsr r r rE � s z,ModuleCommand.UpdateSubCommand.run_on_moduleNrF r r r r �UpdateSubCommand� s rg c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.RemoveSubCommand)�remove�erasez3remove installed module profiles and their packagesc C s0 | j j}d|_d|_d|_d|_d|_d|_d S )NTF)r r5 Z allow_erasingr6 Zfresh_metadatarP rQ r7 r8 r r r r9 � s z(ModuleCommand.RemoveSubCommand.configurec C s� | j �| jj�}| jjr�| �� }| �|t� �\}}| �| jj � � |�\}}| jj � � � � j|d�}| jj � � � � j|d�}|D ]>}||v r�td��|�} t�| � q~| jjj|| jjjd� q~|s�d S t�tjj|d�� d S )N�r z0Package {} belongs to multiple modules, skipping)Z clean_deps)rW )r rh r r �allr* r- r r r$ ZgetModulePackages�sackr% r? �filtermr r" r r! Zgoalri rU Z clean_requirements_on_remover[ r rA rT ) r Zskipped_groupsr' Zremove_names_from_specr( Z keep_namesZ remove_queryZ keep_query�pkgrD r r r rE � s. � �� �z,ModuleCommand.RemoveSubCommand.run_on_moduleNrF r r r r �RemoveSubCommand� s ro c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.SwitchToSubCommand)z switch-toz7switch a module to a stream and distrosync rpm packagesc C s. | j j}d|_d|_d|_d|_d| jj_d S r3 ) r r5 r6 r7 rP rQ r rU Zmodule_stream_switchr8 r r r r9 s z*ModuleCommand.SwitchToSubCommand.configurec C st z | j j| jj| jjjd� W nP tjj yn } z4| jjjrL|j sH|j rL|�t � t|�� W Y d }~n d }~0 0 d S )N)rV )r Z switch_tor r r rU rV r rA rT rW rX r r[ r\ r] r r r rE " s z.ModuleCommand.SwitchToSubCommand.run_on_moduleNrF r r r r �SwitchToSubCommand s rp c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ProvidesSubCommand)Zprovideszlist modular packagesc C s | j j}d|_d|_d S r3 r4 r8 r r r r9 0 s z*ModuleCommand.ProvidesSubCommand.configurec C s | j �| jj�}|r t|� d S r )r Z_what_providesr r r@ rL r r r rE 5 s z.ModuleCommand.ProvidesSubCommand.run_on_moduleNrF r r r r �ProvidesSubCommand+ s rq c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z!ModuleCommand.RepoquerySubCommand)Z repoqueryz#list packages belonging to a modulec C s | j j}d|_d|_d S r3 r4 r8 r r r r9 ? s z+ModuleCommand.RepoquerySubCommand.configurec C s� t � }| jjD ] }| j�|�\}}|�|� q| �|t � �\}}t � }| jjsV| jjs�| j j � � �� j |d�}|D ]} |� t| �� qr| jjr�| j j � � �� j |d�}|D ]} |� t| �� q�d�t|��} t| � d S )N)Z nevra_strictrj � )r r r r r r& r- � availabler? r rl r% rm r, r\ rf �sortedr@ ) r r' r r) r( Znames_from_specZspec_artifactsZpackage_stringsr% rn rC r r r rE D s$ �z/ModuleCommand.RepoquerySubCommand.run_on_moduleNrF r r r r �RepoquerySubCommand: s ru )r zInteract with Modules.c s> t t| ��� � � fdd�| jD �}d | _dd� |D �| _d S )Nc 3 s | ]}|� �V qd S r r )�.0�subcmd�r r r � <genexpr>d � z)ModuleCommand.__init__.<locals>.<genexpr>c S s i | ]}|j D ] }||�qqS r �rG �rv rw �aliasr r r � <dictcomp>f s z*ModuleCommand.__init__.<locals>.<dictcomp>)r r r �SUBCMDSrw �_subcmd_name2obj)r r Z subcmd_objsr rx r r b s �zModuleCommand.__init__c C s |� � }|jdddtd�d� |jdddtd�d� |jd d dtd �d� |jd d dtd�d� |jdddtd�d� |jdddtd�d� g }g }t| jdd� d�D ]2}|�|jd � |�d�|jd |jp�d�� q�|jdd |d d � |�d � |jd d!d"td#�d$� d S )%Nz --enabledr; � store_truezshow only enabled modules)�dest�action�helpz --disabledr> zshow only disabled modulesz --installedr? z'show only installed modules or packagesz --profilerK zshow profile contentz --availablers z show only available packagesz--allrk zremove all modular packagesc S s | j d S �Nr r{ )�xr r r �<lambda>~ rz z-ModuleCommand.set_argparser.<locals>.<lambda>)�keyr z{}: {}r rw rS z<modular command>rr )�nargs�choices�metavarr� r z module-spec�*zModule specification)r� r� r� ) � add_mutually_exclusive_group� add_argumentr rt r �appendrG r" rH rf )r �parserZnarrowsZsubcommand_choicesZsubcommand_helprw r r r � set_argparseri sH ���� �� � �zModuleCommand.set_argparserc C sX z| j | jjd | _W n& ttfy> | jj�� t�Y n0 | j| j_| j�� d S r� ) r� r rw r �KeyErrorr Z optparser� print_usager9 �r r r r r9 � s zModuleCommand.configurec C s | � � | j�� d S r )�check_required_argumentrw rE r� r r r �run� s zModuleCommand.runc C sR dd� | j D �}| jjd |vrN| jjsNttd��tjj | jj | jjd ���d S )Nc S s g | ]}|j D ]}|�qqS r r{ r| r r r � <listcomp>� s �z9ModuleCommand.check_required_argument.<locals>.<listcomp>r z{} {} {}: too few arguments) �SUBCMDS_NOT_REQUIRED_ARGr rw r r r r" r �utilZ MAIN_PROGZcommand)r Znot_required_argumentr r r r� � s � ��z%ModuleCommand.check_required_argument) r. r/ r0 r �Commandr rI rM r_ ra rc re rg ro rp rq ru r r� rG r rH r r� r9 r� r� r1 r r r r r % s0 '"% � r )Z __future__r Zdnf.clir r Zdnf.i18nr Zdnf.module.exceptionsr Zdnf.utilr r �sys�osr+ r< Zdnf.module.module_baseZdnf.exceptionsr� r r r r r �<module> s __pycache__/deplist.cpython-39.pyc 0000644 00000001643 15025176534 0013065 0 ustar 00 a �!c� � @ sP d dl mZ d dl mZ d dl mZ d dlmZ d dlmZ G dd� de�ZdS ) � )�print_function)�absolute_import)�unicode_literals)�_)�RepoQueryCommandc @ s$ e Zd ZdZdZed�Zdd� ZdS )�DeplistCommandz< The command is alias for 'dnf repoquery --deplist' )�deplistz`[deprecated, use repoquery --deplist] List package's dependencies and what packages provide themc C s t �| � d| j_d S )NT)r � configureZoptsr )�self� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/deplist.pyr " s zDeplistCommand.configureN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr r r r r r s r N) Z __future__r r r Zdnf.i18nr Zdnf.cli.commands.repoqueryr r r r r r �<module> s __pycache__/upgrademinimal.cpython-39.opt-1.pyc 0000644 00000002161 15025176534 0015352 0 ustar 00 a �!c � @ sD d dl mZ d dl mZ d dlmZ d dlmZ G dd� de�ZdS )� )�absolute_import)�unicode_literals)�_)�UpgradeCommandc @ s$ e Zd ZdZdZed�Zdd� ZdS )�UpgradeMinimalCommandzSA class containing methods needed by the cli to execute the check command. )zupgrade-minimalzupdate-minimalzup-minzWupgrade, but only 'newest' package match which fixes a problem that affects your systemc C sR t �| � d| _t| jj| jj| jj| jj| jj | jj | jj | jj g�sNd| _ d S )NT)r � configureZupgrade_minimal�anyZoptsZbugfixZ enhancementZ newpackageZsecurityZadvisoryZbugzillaZcvesZseverityZ all_security)�self� r �C/usr/lib/python3.9/site-packages/dnf/cli/commands/upgrademinimal.pyr " s �z UpgradeMinimalCommand.configureN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr r r r r r s r N)Z __future__r r Zdnf.i18nr Zdnf.cli.commands.upgrader r r r r r �<module> s __pycache__/makecache.cpython-39.opt-1.pyc 0000644 00000002434 15025176534 0014260 0 ustar 00 a �!cm � @ sx d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dlZd dl Zd dl Zd dl Z e � d�Z G dd� dej�ZdS ) � )�absolute_import)�unicode_literals)�commands)�_N�dnfc @ s, e Zd ZdZed�Zedd� �Zdd� ZdS )�MakeCacheCommand)Z makecacheZmczgenerate the metadata cachec C s, | j dddd� | j dddgdtjd� d S )Nz--timer� store_true� timer_opt)�action�dest�timer�?)�nargs�choices�metavar�help)� add_argument�argparse�SUPPRESS)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/makecache.py� set_argparser' s �z MakeCacheCommand.set_argparserc C s2 | j jd up| j j}td�}t�|� | j�|�S )Nz*Making cache files for all metadata files.)Zoptsr r r �logger�debug�baseZ update_cache)�selfr �msgr r r �run. s zMakeCacheCommand.runN) �__name__� __module__� __qualname__�aliasesr Zsummary� staticmethodr r r r r r r # s r )Z __future__r r Zdnf.clir Zdnf.i18nr r r Zdnf.exceptionsZdnf.utilZloggingZ getLoggerr ZCommandr r r r r �<module> s __pycache__/clean.cpython-39.pyc 0000644 00000010301 15025176534 0012472 0 ustar 00 a �!cn � @ s� d dl mZ d dl mZ d dlmZ d dlmZmZ d dlm Z d dlZ d dl Z d dl Z d dl Z d dlZ d dlZd dlZd dlZd dlZe�d�Zg d�d gd gd gg d �d �Zdd� Zdd� Zdd� Zdd� ZG dd� dej�ZdS )� )�absolute_import)�unicode_literals)�commands)�_�P_)�miscN�dnf)�metadata�dbcache� expire-cache�packagesr r )r r r )r r r r �allc c sN t �| �D ]>\}}}t j�|| �}|D ] }t j�||�}t j�|�V q&q dS )z:Traverse dirpath recursively and yield relative filenames.N)�os�walk�path�relpath�join�normpath)�dirpath�root�dirs�files�base�fr � r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/clean.py�_tree1 s r c s � fdd�| D �S )z5Yield those filenames that match any of the patterns.c 3 s( | ] }� D ]}t �||�r |V q qd S �N��re�match)�.0r �p��patternsr r � <genexpr>< � z_filter.<locals>.<genexpr>r )r r$ r r# r �_filter: s r' c C sH d}|D ]:}t j�| |�}t�tjjtd�|� t � |� |d7 }q|S )z(Remove the given filenames from dirpath.r zRemoving file %s� ) r r r �logger�logr �loggingZDDEBUGr r Zunlink_f)r r �countr r r r r �_clean? s r- c s0 t jjd � � fdd�| D �}tdd� |D ��S )z:Return the repo IDs that have some cached metadata around.r c 3 s | ]}t �� |�V qd S r r )r! r �Zmetapatr r r% M r& z _cached_repos.<locals>.<genexpr>c s s | ]}|r|� d �V qdS )ZrepoidN)�group)r! �mr r r r% N r& )r �repo� CACHE_FILES�set)r �matchesr r. r � _cached_reposJ s r5 c @ s0 e Zd ZdZdZed�Zedd� �Zdd� Z dS ) � CleanCommandzSA class containing methods needed by the cli to execute the clean command. )Zcleanzremove cached datac C s | j ddt�� td�d� d S )N�type�+zMetadata type to clean)�nargs�choices�help)� add_argument� _CACHE_TYPES�keysr )�parserr r r � set_argparserY s �zCleanCommand.set_argparserc C s� | j jj}tj�|d�}tj�|d�}tj�| j jjd�}�z<|�� |� �� |�� t dd� | j j D ��}t t |��}t�tdd�|� �� d|v r�t|�}| j jj�|� |�d� t�td�� dd � |D �}t|t||��} t�td d | �| � W d � W d � W d � W d S 1 �s,0 Y W d � n1 �sL0 Y W d � n1 �sl0 Y W q: tjj �y� } z@| j jj �s�td �| j } t�| � t � d � n| �W Y d } ~ q:d } ~ 0 0 q:d S )NTc s s | ]}t | D ] }|V qqd S r )r= )r! �c�tr r r r% g r& z#CleanCommand.run.<locals>.<genexpr>zCleaning data: � r zCache was expiredc S s g | ]}t jj| �qS r )r r1 r2 )r! rB r r r � <listcomp>q r& z$CleanCommand.run.<locals>.<listcomp>z%d file removedz%d files removedz*Waiting for process with pid %d to finish.� )!r Zconf�cachedirr �lockZbuild_metadata_lockZbuild_download_lockZbuild_rpmdb_lockZ persistdirr3 Zoptsr7 �listr r) �debugr r r5 Z_repo_persistorZexpired_to_add�update�remove�infor- r' r � exceptionsZ LockErrorZ exit_on_lock�pid�time�sleep) �selfrF Zmd_lockZ download_lockZ rpmdb_lock�typesr Zexpiredr$ r, �e�msgr r r �run_ s0 � zCleanCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr@ rU r r r r r6 Q s r6 ) Z __future__r r Zdnf.clir Zdnf.i18nr r Zdnf.yumr r Zdnf.exceptionsZdnf.lockZ dnf.loggingZdnf.repor+ r r rO Z getLoggerr) r= r r' r- r5 ZCommandr6 r r r r �<module> s2 � __pycache__/remove.cpython-39.opt-1.pyc 0000644 00000007557 15025176534 0013667 0 ustar 00 a �B�f� � @ s� d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ d dl Z d dl Z d dl Z d dl Z d dlZe�d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)�commands)�_)� OptionParserN�dnfc @ sb e Zd ZdZejejejejejejd�Zde e� � � Z e d�Z edd� �Zdd� Zd d � Zd S ) � RemoveCommandzRemove command.)zremove-nz remove-naz remove-nevrazerase-nzerase-naz erase-nevra)�removeZeraseZrmz-remove a package or packages from your systemc C sf | � � }|jdddtd�d� |jddtjd� |jddtd �d� | jd d td �tjtd �d� d S )Nz --duplicates� store_true� duplicatedzremove duplicated packages)�action�dest�helpz --duplicated)r r z--oldinstallonlyz*remove installonly packages over the limitZpackages�*zPackage to removeZPACKAGE)�nargsr r �metavar)� add_mutually_exclusive_group� add_argumentr �argparse�SUPPRESSr ZParseSpecGroupFileCallback)�parserZmgroup� r �;/usr/lib/python3.9/site-packages/dnf/cli/commands/remove.py� set_argparser0 s" �����zRemoveCommand.set_argparserc C s^ | j j}d|_d|_d|_| jjr*d|_n0tj j rN| jj rNd|_d|_ d|_ n d|_ d|_d S )NTF)Zcli�demandsZ resolvingZ root_userZsack_activation�optsr Zavailable_reposr �base� WITH_MODULES� grp_specsZfresh_metadataZ allow_erasing)�selfr r r r � configure? s zRemoveCommand.configurec C sB g }| j j| jv r"| j| j j g}| j j| j j7 _d}| j j�r<| jj�� }| j� |� � �}|�� � |�}|s�t � td�� d S |�� �� D ]�\\}}}t|�dk r�q�|jdd� z| j�t|d �� W nF tjj�y d} td�} t �| | jjj�t|d ��| � Y n0 |d d � D ]} | j� | � �q"q�d S | j j �r�| jj�� }| j� |� � �� d �}| jj� � } | d u�r�|j | j!| j"| j#d �} | �r�|� | �}|�r�|D ]} | j� | � �q�nt � td �� d S | j j$�r|�r| j j$D ]&}td �} t �| | jjj�|�� �q�n�| j j$�r�tjj%�rftj&j'�(| j�}|�)| j j$�}t| j j$�t|�k�rnd}n| j j$}|�r�|D ]8}z| j�*|g��r�d}W n tjj+�y� Y n0 �qx| j jD ]n}z| jj)||d� W nN tjj,�y } z0d�-|j.| jjj�|��} t � | � W Y d }~nd }~0 0 d}�q�|�s>t �td�� d S )NFz)No duplicated packages found for removal.� T)�reverser � z%Installed package %s%s not available.� ���)�epoch�version�releasez.No old installonly packages found for removal.zNot a valid form: %s)�formsz{}: {}z No packages marked for removal.)/r Zcommand� nevra_formsZ pkg_specs� filenamesr r Zsack�queryZ_get_installonly_queryZ installed� difference�logger�infor Z_na_dict�items�len�sortZ reinstall�strr � exceptionsZPackagesNotAvailableErrorZwarning�outputZtermZboldZpackage_removeZoldinstallonlyZlatestZget_running_kernel�filterr& r' r( r r �module� module_baseZ ModuleBaser Zenv_group_remove�ErrorZ MarkingError�format�value)r r) Zdone�qZinstonlyZdups�nameZarchZ pkgs_listZxmsg�msg�pkgZkernelZrunning_installonlyZgrp_specr8 Z skipped_grps�groupZpkg_spec�er r r �runP s� ( � zRemoveCommand.runN)�__name__� __module__� __qualname__�__doc__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr* �tuple�keys�aliasesr Zsummary� staticmethodr r rB r r r r r # s � r )Z __future__r r Zdnf.clir Zdnf.i18nr Zdnf.cli.option_parserr Zdnf.baser r rG Zdnf.exceptionsZloggingZ getLoggerr. ZCommandr r r r r �<module> s __pycache__/history.cpython-39.pyc 0000644 00000026315 15025176534 0013125 0 ustar 00 a �!c%F � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlZd dlmZmZ d dl m Z d dl m Z m Z d dl Zd dlZd dlZd dlZd dlZd dlZd dlZe�d�ZG d d � d e j�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�_�ucd)�commands)�TransactionReplay�serialize_transaction�dnfc s� e Zd ZdZdZed�Zg d�Z� fdd�Ze dd� �Z d d � Z d d � Z d d� Z dd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Z� ZS )#�HistoryCommandzUA class containing methods needed by the cli to execute the history command. )�historyZhistz(display, or use, the transaction history)�list�info�redo�replay�rollback�store�undo� userinstalledc s t t| �j|i |�� d| _d S )NF)�superr �__init__�_require_one_transaction_id)�self�args�kw�� __class__� �</usr/lib/python3.9/site-packages/dnf/cli/commands/history.pyr 4 s zHistoryCommand.__init__c C s� | j dddd�tjd d�tjdd � ��d� | j d d d d � | j d dd td�d� | j dd td�d � | j dd td�d � | j dd td�d � | j ddddd� | j ddd d d� d S ) N�transactions_action�?ZCOMMANDz$Available commands: {} (default), {}r z, � )�nargs�metavar�helpz --reverse� store_truez$display history list output reversed)�actionr$ z-oz--outputz<For the store command, file path to store the transaction to)�defaultr$ z--ignore-installedzXFor the replay command, don't check for installed packages matching those in transactionz--ignore-extraszRFor the replay command, don't check for extra packages pulled into the transactionz--skip-unavailablezYFor the replay command, skip packages that are not available or have missing dependencies� transactions�*Z TRANSACTIONz�For commands working with history transactions, Transaction ID (<number>, 'last' or 'last-<number>' for one transaction, <transaction-id>..<transaction-id> for a range)�transaction_filenameZTRANSACTION_FILEzEFor the replay command, path to the stored transaction file to replay)� add_argument�formatr �_CMDS�joinr )�parserr r r � set_argparser9 s6 ��� ���� � �z HistoryCommand.set_argparserc C s, | j js| jd | j _n0| j j| jvrH| j j�d| j j� | jd | j _td��| j j�| _| jj }| j jdk�r| j js�t j� td���t | j j�dkr�t j� td���t j�| j jd �| j _g | j _d|_d|_d|_d| jj_d| jj_t jj�| j| j� n�| j jd k�r6d| _| j j�s�t j� td ���n�| j jd v �r�d|_d|_d|_d| _| j j�s�td �}t� |� t j� |��n,t | j j�dk�r�t� | j� t j� | j��d|_t jj�| j| j� nd|_ d|_ | jj jd k�r(t � | jj jt j!��s(td | jj j �}t� |� t j� |��d S )Nr zUFound more than one transaction ID. '{}' requires one transaction ID or package name.r z No transaction file name given.r! z6More than one argument given as transaction file name.TFr z(No transaction ID or package name given.)r r r z:memory:z+You don't have access to the history DB: %s)"�optsr r- r( �insertr r, � _require_one_transaction_id_msg�cli�demandsr �CliError�len�os�path�abspathr* Zavailable_reposZ resolvingZ root_user�base�confZ clean_requirements_on_removeZinstall_weak_depsr Z _checkGPGKeyr �logger�criticalZfresh_metadataZsack_activationr �access�R_OK)r r5 �msgr r r � configureU s\ � & zHistoryCommand.configurec C s� t |tjj�rv| jjdkr2| jj\}td�| fS | jjdkrv| jjd dkrV| jjn| jjdd� \}td�| fS tjj j � | |�S ) z.Get suggestions for resolving the given error.r zVCannot undo transaction %s, doing so would result in an inconsistent package database.r r �forcer! NzZCannot rollback transaction %s, doing so would result in an inconsistent package database.) � isinstancer � exceptionsZTransactionCheckErrorr1 r r( r r4 r �Command�get_error_output)r �errorZid_r r r rG � s � ��z HistoryCommand.get_error_outputc C s: | � |�}t|�}t| j|dd| jjd�| _| j�� d S )NT��data�ignore_installed� ignore_extras�skip_unavailable)�_history_get_transactionr r r; r1 rM r �run)r �extcmds�oldrJ r r r � _hcmd_redo� s �zHistoryCommand._hcmd_redoc C sD |st j�td���| jj�|�}|s@t j�td��|d ���|S )NzNo transaction ID givenz Transaction ID "{0}" not found.r )r r4 r6 r r; r rQ r, �r rP rQ r r r �_history_get_transactions� s z(HistoryCommand._history_get_transactionsc C s. | � |�}t|�dkr&tj�td���|d S )Nr! z#Found more than one transaction ID!r )rT r7 r r4 r6 r rS r r r rN � s z'HistoryCommand._history_get_transactionc C s | � |�}| �|� d S )N)rN �_revert_transactionrS r r r � _hcmd_undo� s zHistoryCommand._hcmd_undoc C s� | � |�}| jj�� }d }|j|jkr�| jj�tt|jd |jd ���D ]X}|jrht � t d�|j� n|j r�t � t d�|j� |d u r�t jj�|�}qJ|�|� qJ| �|� d S )Nr! z-Transaction history is incomplete, before %u.z,Transaction history is incomplete, after %u.)rN r; r �last�tidrQ r �rangeZaltered_lt_rpmdbr= Zwarningr Zaltered_gt_rpmdbr ZdbZMergedTransactionWrapper�mergerU )r rP rQ rW Z merged_trans�transr r r �_hcmd_rollback� s ( z HistoryCommand._hcmd_rollbackc C s dddddddddd d d � }t |�}d D ]�}|�|g �D ]�}||d |d < |d dkrp|�dd �dkrpd|d< |d d kr�d|v r�t�|d �}|jtjgd�d }| jjj� |j |j |� � d d �}t j�|�|d< |�d�tjkr8d |d< q8q(t| j|dd| jjd�| _| j�� d S )N�Removed�Install� Downgraded� Downgrade�Upgraded�Upgrade� Reinstall� Reinstalled� Obsoleted� Reason Change) r^ r] rb ra r` r_ rd rc re ZObsoleterf )Zrpms�groupsZ environmentsr& �reasonZcleanZ dependency�nevra)Zformsr r! Zrepo_idTrI )r �get�hawkeyZSubjectZget_nevra_possibilitiesZ FORM_NEVRA�outputr ZswdbZ resolveRPMTransactionItemReason�nameZarch�tids�libdnfZ transactionZ TransactionItemReasonToStringZSYSTEM_REPO_NAMEr r; r1 rM r rO ) r r[ Z action_maprJ Z content_typeZtiZsubjri rh r r r rU � sJ � � �z"HistoryCommand._revert_transactionc C s: t | j�� �}| j�|dd�}|dkr6tj�td���dS )z&Execute history userinstalled command.zPackages installed by userri r zNo packages to listN) �tupler; Ziter_userinstalledrl ZlistPkgsr r4 r6 r )r ZpkgsZn_listedr r r �_hcmd_userinstalled s z"HistoryCommand._hcmd_userinstalledc s� � fdd�}t � }t � }� jjD �]�}d|v �rRz|�dd�\}}W n. tyr t�td��|�� t j j �Y n0 td�}z ||�}W n. ty� t�t|��|�� t j j �Y n0 z ||�}W n. ty� t�t|��|�� t j j �Y n0 � j �r||k�rt�� j � t j j �||k�r.|| }}|�||f� |�t||d �� q z|�||�� W q t�y� � jj�|g�}|�r�|�|� n4td��|�} � j �r�t�| � t j j �n t�| � Y q 0 q t|d d �|fS ) z0Convert commandline arguments to transaction idsc sJ | dkrd} n| � d�r$| dd � } t| �}|dkrF|� jj�� j7 }|S )NrW �0zlast-� r )� startswith�intrl r rW rX )�sZtransaction_id�r r r �str2transaction_id s z@HistoryCommand._args2transaction_ids.<locals>.str2transaction_idz..� zWInvalid transaction ID range definition '{}'. Use '<transaction-id>..<transaction-id>'.zNCan't convert '{}' to transaction ID. Use '<number>', 'last', 'last-<number>'.r! z8No transaction which manipulates package '{}' was found.T��reverse)�setr1 r( �split� ValueErrorr= r> r r, r r4 r6 r r3 �add�updaterY rl r �searchr �sorted) r rx rn � merged_tids�tZbegin_transaction_idZend_transaction_idZcant_convert_msgZtransact_ids_from_pkgnamerA r rw r �_args2transaction_ids s^ �� � � z$HistoryCommand._args2transaction_idsc C sP | j j}|dkrDt| j| j j| j j| j j| j jd�| _| j� � �n| � � \}}|dkr||sd| j j s|| j j || j jd� �n�|dkr�|s�| j j s�| j �|| j j |� �n�|dkr�| �|� �n�|dkr�| �|� �nx|dkr�| �|� �nb|d kr�| �� �nN|d k�rL| �|�}t|�}z�| j j d u�r2| j j nd }| jjj�sN| jjj�s�tj�|��r�t d �� |�}| jjj�s�| jj j d � |�d� |�d��s�t t d�� |�� W d S t |d��,}t!j"||ddd� |�#d� W d � n1 �s�0 Y t t d�� |�� W n@ t$�yJ } z&t%j&�'t d�� t(| ����W Y d } ~ n d } ~ 0 0 d S )Nr )�filenamerK rL rM r rz r r r r r r ztransaction.jsonz{} exists, overwrite?z {} [y/N]: z {} [Y/n]: )rA Zdefaultyes_msgz Not overwriting {}, exiting.�wrs T)�indentZ sort_keys� zTransaction saved to {}.z Error storing transaction: {}))r1 r r r; r* rK rL rM r rO r� r( rl ZhistoryListCmdr{ ZhistoryInfoCmdrV rR r\ rq rN r r< ZassumenoZ assumeyesr8 r9 �isfiler r, Z userconfirm�print�open�json�dump�write�OSErrorr r4 r6 �str) r Zvcmdrn r� rX rJ r� rA �f�er r r rO M sR � &� *zHistoryCommand.runc C s | j jdvrd S | j�� d S )N�r r r r )r1 r r Zpost_transactionrw r r r � run_resolved s zHistoryCommand.run_resolvedc C sT | j jdvrd S | j�� }|rPt�tjjt d�� |D ]}t�tjjd| � q6d S )Nr� zEWarning, the following problems occurred while running a transaction:z ) r1 r r Z get_warningsr= �logr �loggingZWARNINGr )r �warningsr� r r r �run_transaction� s �z HistoryCommand.run_transaction)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr- r � staticmethodr0 rB rG rR rT rN rV r\ rU rq r� rO r� r� � __classcell__r r r r r * s( = 0@2r )Z __future__r r r ro rk Zdnf.i18nr r Zdnf.clir Zdnf.transaction_srr r r Zdnf.exceptionsZdnf.transactionZdnf.utilr� r� r8 Z getLoggerr= rF r r r r r �<module> s __pycache__/upgrademinimal.cpython-39.pyc 0000644 00000002161 15025176534 0014413 0 ustar 00 a �!c � @ sD d dl mZ d dl mZ d dlmZ d dlmZ G dd� de�ZdS )� )�absolute_import)�unicode_literals)�_)�UpgradeCommandc @ s$ e Zd ZdZdZed�Zdd� ZdS )�UpgradeMinimalCommandzSA class containing methods needed by the cli to execute the check command. )zupgrade-minimalzupdate-minimalzup-minzWupgrade, but only 'newest' package match which fixes a problem that affects your systemc C sR t �| � d| _t| jj| jj| jj| jj| jj | jj | jj | jj g�sNd| _ d S )NT)r � configureZupgrade_minimal�anyZoptsZbugfixZ enhancementZ newpackageZsecurityZadvisoryZbugzillaZcvesZseverityZ all_security)�self� r �C/usr/lib/python3.9/site-packages/dnf/cli/commands/upgrademinimal.pyr " s �z UpgradeMinimalCommand.configureN)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr r r r r r s r N)Z __future__r r Zdnf.i18nr Zdnf.cli.commands.upgrader r r r r r �<module> s __pycache__/module.cpython-39.opt-1.pyc 0000644 00000034203 15025176534 0013643 0 ustar 00 a �!c�A � @ s� d dl mZ d dlmZmZ d dlmZ d dlmZ d dl m Z d dl Z d dl Z d dl Z d dlZd dlZd dlZ d dlZ G dd� dej�ZdS ) � )�print_function)�commands�CliError)�_)�NoModuleException)�loggerNc s* e Zd ZG dd� dej�ZG dd� de�ZG dd� de�ZG dd� de�ZG d d � d e�Z G d d � d e�Z G d d� de�Z G dd� de�Z G dd� de�Z G dd� de�ZG dd� de�ZG dd� de�Zeeee e e e e eeeh ZehZdZed�Z� fdd �Zd d � Zd d � Zd!d"� Zd#d$� Z� ZS )%� ModuleCommandc s, e Zd Z� fdd�Zdd� Zdd� Z� ZS )zModuleCommand.SubCommandc s( t tj| ��|� tjj�| j�| _d S �N) �superr � SubCommand�__init__�dnf�module� module_baseZ ModuleBase�base)�self�cli�� __class__� �;/usr/lib/python3.9/site-packages/dnf/cli/commands/module.pyr ( s z!ModuleCommand.SubCommand.__init__c C s� t � }| jjD ]�}| j�|�\}}|d u r,q|jr8|jnd}|jrH|jnd}|jr\|jdksb|jrvt � t d�� |�� |j r�|j nd}| jj�||dd|�}|�|� q|S )N� ���zjOnly module name, stream, architecture or profile is used. Ignoring unneeded information in argument: '{}')�set�opts� module_specr � _get_modules�name�stream�version�contextr �infor �format�archr �_moduleContainer�query�update) r �modules_from_specsr �__Znsvcapr r r# �modulesr r r �#_get_modules_from_name_stream_specs, s � z<ModuleCommand.SubCommand._get_modules_from_name_stream_specsc C s~ t � }t � }|D ](}||vr| jj�|�r|�|�� � q|D ]6}t�|�}|jtj gd�D ]}|j r\|� |j � q\q>||fS )N)Zforms) r r r$ ZisModuleActiver& Z getArtifacts�hawkeyZSubjectZget_nevra_possibilitiesZ FORM_NEVRAr �add) r Z use_modulesZ skip_modulesZ artifactsZ pkg_namesr ZartifactZsubjZ nevra_objr r r �_get_module_artifact_names> s � z3ModuleCommand.SubCommand._get_module_artifact_names)�__name__� __module__� __qualname__r r* r- � __classcell__r r r r r & s r c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ListSubCommand)�listz,list all module streams, profiles and statesc C s | j j}d|_d|_d S �NT�r �demands�available_repos�sack_activation�r r5 r r r � configureR s z&ModuleCommand.ListSubCommand.configurec C s� | j }| jjr&|�| jjtjjj�}nV| jj rF|�| jjtjjj �}n6| jj rf|�| jjtjjj �}n|�| jjtjjj �}|r�t|� d S | jjr�td�}tj�|��d S �NzNo matching Modules to list)r r �enabledZ_get_brief_descriptionr �libdnfr �ModulePackageContainerZModuleState_ENABLED�disabledZModuleState_DISABLED� installedZModuleState_INSTALLEDZModuleState_UNKNOWN�printr r � exceptions�Error)r Zmods�output�msgr r r � run_on_moduleW s0 ����z*ModuleCommand.ListSubCommand.run_on_moduleN�r. r/ r0 �aliasesr �summaryr9 rE r r r r �ListSubCommandM s rI c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.InfoSubCommand)r! z)print detailed information about a modulec C s | j j}d|_d|_d S r3 r4 r8 r r r r9 t s z&ModuleCommand.InfoSubCommand.configurec C sf | j jr| j�| j j�}n*| j jr4| j�| j j�}n| j�| j j�}|rRt|� nt j � t d���d S r: ) r �verboser Z_get_full_infor �profileZ_get_info_profilesZ _get_infor@ r rA rB r �r rC r r r rE y s z*ModuleCommand.InfoSubCommand.run_on_moduleNrF r r r r �InfoSubCommando s rM c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.EnableSubCommand)�enablezenable a module streamc C s$ | j j}d|_d|_d|_d|_d S r3 �r r5 r6 r7 � resolving� root_userr8 r r r r9 � s z(ModuleCommand.EnableSubCommand.configurec C s� z| j �| jj� W nn tjjy� } zR| jjj r`|j s>|j rB|�|j r`|j d t jjjkr`|�t�t|�� W Y d }~n d }~0 0 d S �N� )r rN r r r rA � MarkingErrorsr �conf�strict�no_match_group_specs�error_group_specs�module_depsolv_errorsr< r r= �!ModuleErrorType_ERROR_IN_DEFAULTSr �error�str�r �er r r rE � s �z,ModuleCommand.EnableSubCommand.run_on_moduleNrF r r r r �EnableSubCommand� s r_ c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.DisableSubCommand)�disablez%disable a module with all its streamsc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z)ModuleCommand.DisableSubCommand.configurec C s� z| j �| jj� W nn tjjy� } zR| jjj r`|j s>|j rB|�|j r`|j d t jjjkr`|�t�t|�� W Y d }~n d }~0 0 d S rR )r r` r r r rA rT r rU rV rW rX rY r< r r= rZ r r[ r\ r] r r r rE � s �z-ModuleCommand.DisableSubCommand.run_on_moduleNrF r r r r �DisableSubCommand� s ra c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ResetSubCommand)�resetzreset a modulec C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z'ModuleCommand.ResetSubCommand.configurec C sd z| j �| jj� W nJ tjjy^ } z.| jjj r<|j r<|�t � t |�� W Y d }~n d }~0 0 d S r )r rb r r r rA rT r rU rV rW r r[ r\ r] r r r rE � s z+ModuleCommand.ResetSubCommand.run_on_moduleNrF r r r r �ResetSubCommand� s rc c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.InstallSubCommand)�installz/install a module profile including its packagesc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z)ModuleCommand.InstallSubCommand.configurec C sr z | j �| jj| jjj� W nP tjj yl } z4| jjjrJ|j sF|j rJ|�t � t|�� W Y d }~n d }~0 0 d S r )r rd r r r rU rV r rA rT rW rX r r[ r\ r] r r r rE � s z-ModuleCommand.InstallSubCommand.run_on_moduleNrF r r r r �InstallSubCommand� s re c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.UpdateSubCommand)r& z0update packages associated with an active streamc C s$ | j j}d|_d|_d|_d|_d S r3 rO r8 r r r r9 � s z(ModuleCommand.UpdateSubCommand.configurec C s& | j �| jj�}|r"td�|���d S )Nz, )r Zupgrader r r �join)r Z module_specsr r r rE � s z,ModuleCommand.UpdateSubCommand.run_on_moduleNrF r r r r �UpdateSubCommand� s rg c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.RemoveSubCommand)�remove�erasez3remove installed module profiles and their packagesc C s0 | j j}d|_d|_d|_d|_d|_d|_d S )NTF)r r5 Z allow_erasingr6 Zfresh_metadatarP rQ r7 r8 r r r r9 � s z(ModuleCommand.RemoveSubCommand.configurec C s� | j �| jj�}| jjr�| �� }| �|t� �\}}| �| jj � � |�\}}| jj � � � � j|d�}| jj � � � � j|d�}|D ]>}||v r�td��|�} t�| � q~| jjj|| jjjd� q~|s�d S t�tjj|d�� d S )N�r z0Package {} belongs to multiple modules, skipping)Z clean_deps)rW )r rh r r �allr* r- r r r$ ZgetModulePackages�sackr% r? �filtermr r" r r! Zgoalri rU Z clean_requirements_on_remover[ r rA rT ) r Zskipped_groupsr' Zremove_names_from_specr( Z keep_namesZ remove_queryZ keep_query�pkgrD r r r rE � s. � �� �z,ModuleCommand.RemoveSubCommand.run_on_moduleNrF r r r r �RemoveSubCommand� s ro c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.SwitchToSubCommand)z switch-toz7switch a module to a stream and distrosync rpm packagesc C s. | j j}d|_d|_d|_d|_d| jj_d S r3 ) r r5 r6 r7 rP rQ r rU Zmodule_stream_switchr8 r r r r9 s z*ModuleCommand.SwitchToSubCommand.configurec C st z | j j| jj| jjjd� W nP tjj yn } z4| jjjrL|j sH|j rL|�t � t|�� W Y d }~n d }~0 0 d S )N)rV )r Z switch_tor r r rU rV r rA rT rW rX r r[ r\ r] r r r rE " s z.ModuleCommand.SwitchToSubCommand.run_on_moduleNrF r r r r �SwitchToSubCommand s rp c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z ModuleCommand.ProvidesSubCommand)Zprovideszlist modular packagesc C s | j j}d|_d|_d S r3 r4 r8 r r r r9 0 s z*ModuleCommand.ProvidesSubCommand.configurec C s | j �| jj�}|r t|� d S r )r Z_what_providesr r r@ rL r r r rE 5 s z.ModuleCommand.ProvidesSubCommand.run_on_moduleNrF r r r r �ProvidesSubCommand+ s rq c @ s( e Zd ZdZed�Zdd� Zdd� ZdS )z!ModuleCommand.RepoquerySubCommand)Z repoqueryz#list packages belonging to a modulec C s | j j}d|_d|_d S r3 r4 r8 r r r r9 ? s z+ModuleCommand.RepoquerySubCommand.configurec C s� t � }| jjD ] }| j�|�\}}|�|� q| �|t � �\}}t � }| jjsV| jjs�| j j � � �� j |d�}|D ]} |� t| �� qr| jjr�| j j � � �� j |d�}|D ]} |� t| �� q�d�t|��} t| � d S )N)Z nevra_strictrj � )r r r r r r& r- � availabler? r rl r% rm r, r\ rf �sortedr@ ) r r' r r) r( Znames_from_specZspec_artifactsZpackage_stringsr% rn rC r r r rE D s$ �z/ModuleCommand.RepoquerySubCommand.run_on_moduleNrF r r r r �RepoquerySubCommand: s ru )r zInteract with Modules.c s> t t| ��� � � fdd�| jD �}d | _dd� |D �| _d S )Nc 3 s | ]}|� �V qd S r r )�.0�subcmd�r r r � <genexpr>d � z)ModuleCommand.__init__.<locals>.<genexpr>c S s i | ]}|j D ] }||�qqS r �rG �rv rw �aliasr r r � <dictcomp>f s z*ModuleCommand.__init__.<locals>.<dictcomp>)r r r �SUBCMDSrw �_subcmd_name2obj)r r Z subcmd_objsr rx r r b s �zModuleCommand.__init__c C s |� � }|jdddtd�d� |jdddtd�d� |jd d dtd �d� |jd d dtd�d� |jdddtd�d� |jdddtd�d� g }g }t| jdd� d�D ]2}|�|jd � |�d�|jd |jp�d�� q�|jdd |d d � |�d � |jd d!d"td#�d$� d S )%Nz --enabledr; � store_truezshow only enabled modules)�dest�action�helpz --disabledr> zshow only disabled modulesz --installedr? z'show only installed modules or packagesz --profilerK zshow profile contentz --availablers z show only available packagesz--allrk zremove all modular packagesc S s | j d S �Nr r{ )�xr r r �<lambda>~ rz z-ModuleCommand.set_argparser.<locals>.<lambda>)�keyr z{}: {}r rw rS z<modular command>rr )�nargs�choices�metavarr� r z module-spec�*zModule specification)r� r� r� ) � add_mutually_exclusive_group� add_argumentr rt r �appendrG r" rH rf )r �parserZnarrowsZsubcommand_choicesZsubcommand_helprw r r r � set_argparseri sH ���� �� � �zModuleCommand.set_argparserc C sX z| j | jjd | _W n& ttfy> | jj�� t�Y n0 | j| j_| j�� d S r� ) r� r rw r �KeyErrorr Z optparser� print_usager9 �r r r r r9 � s zModuleCommand.configurec C s | � � | j�� d S r )�check_required_argumentrw rE r� r r r �run� s zModuleCommand.runc C sR dd� | j D �}| jjd |vrN| jjsNttd��tjj | jj | jjd ���d S )Nc S s g | ]}|j D ]}|�qqS r r{ r| r r r � <listcomp>� s �z9ModuleCommand.check_required_argument.<locals>.<listcomp>r z{} {} {}: too few arguments) �SUBCMDS_NOT_REQUIRED_ARGr rw r r r r" r �utilZ MAIN_PROGZcommand)r Znot_required_argumentr r r r� � s � ��z%ModuleCommand.check_required_argument) r. r/ r0 r �Commandr rI rM r_ ra rc re rg ro rp rq ru r r� rG r rH r r� r9 r� r� r1 r r r r r % s0 '"% � r )Z __future__r Zdnf.clir r Zdnf.i18nr Zdnf.module.exceptionsr Zdnf.utilr r �sys�osr+ r< Zdnf.module.module_baseZdnf.exceptionsr� r r r r r �<module> s __pycache__/shell.cpython-39.pyc 0000644 00000017400 15025176534 0012526 0 ustar 00 a �!cl& � @ s� d dl mZ d dlmZmZ d dlZd dlZd dlZd dlZd dl Z d dl Z d dl Z e � d�Z G dd� de�ZG dd� dejej�ZdS ) � )�commands)�_�ucdN�dnfc @ s e Zd ZdZdZdZdZdS )�ShellDemandSheetTN)�__name__� __module__� __qualname__Zavailable_reposZ resolvingZ root_userZsack_activation� r r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/shell.pyr % s r c @ s� e Zd ZdZed�jejjd�Z ddddddddd d d � Z d d � Z e dd� �Z dd� Zdd� Zdd� Zdd� Zd)dd�Zd*dd �Zd+d d �Zd,d d �Zd!d"� Zd-d#d$�Zd.d%d&�Zd/d'd(�ZdS )0� ShellCommand)�shell�shz run an interactive {prog} shell��prog�repo�quitZts_run� transaction�config�resolve�help) r � repository�exitr �run�tsr r � resolvedepr c C s$ t j�| |� tj�| � d| _d S )Nz> )r �Command�__init__�cmd�Cmd�prompt)�self�clir r r r = s zShellCommand.__init__c C s* | j ddtd�td�jtjjd�d� d S )N�script�?ZSCRIPTz Script to run in {prog} shellr )�nargs�metavarr )� add_argumentr �formatr �util�MAIN_PROG_UPPER)�parserr r r � set_argparserB s ��zShellCommand.set_argparserc C sl | j j}t� | j _t|�D ]L}|�d�r*qzt| j j|� W q tyd t| j j|t||�� Y q0 qd S )N�__)r" �demandsr �dir� startswith�getattr�AttributeError�setattr)r! Zdefault_demands�attrr r r � configureH s zShellCommand.configurec C s$ | j jr| �| j j� n| �� d S �N)�optsr# � _run_scriptZcmdloop�r! r r r r U s zShellCommand.runc C s | j �� d | j _| j �� d S r6 )�baseZ_finalize_base� _transaction� fill_sackr9 r r r �_clean[ s zShellCommand._cleanc C sh |r |dkrd S |dkr d}zt �|�}W n | �� Y d S 0 | jjjdd� | jj�|�}|jdkrnd S |j| jv r�t | d| j|j �|dd � � n�| jj � |j�}|d u�r\|| j�}z| jj� ||�}W n t y� Y d S 0 z&t�| jj�|j_|�� |�� W nF tjj�yX } z(t�td �d t|� � W Y d }~d S d }~0 0 n| �� d S ) N� ZEOFr F)Z reset_usager r � �Error:� )�shlex�split�_helpr" � optparserr Zparse_main_argsZcommand�MAPPINGr1 Z cli_commands�getZparse_command_args� SystemExit�copy�deepcopyr. r5 r r � exceptions�Error�logger�errorr r )r! �lineZs_liner7 Zcmd_clsr �er r r �onecmd` s< $ zShellCommand.onecmdNc C s� dd� }|rt |�dkr&| �d� d S |d }t |�dkrB|d nd }|�d�}|dkr�|d |� }||d d � }| jj�|�}|D ]}||||� q�|s�t�td �|� n|||| jj � d S ) Nc S sP |rt || |� n:z td�| t|t| ���� W n t�td�� Y n0 d S )Nz{}: {}zUnsupported key value.)r3 �printr( r1 �strrM �warningr )�key�val�confr r r � print_or_set� s z*ShellCommand._config.<locals>.print_or_set� r r r? �.���z Could not find repository: %s) �lenrD �findr: �repos� get_matchingrM rT r rW ) r! �argsrX rU rV ZperiodZ repo_namer^ r r r r �_config� s$ �zShellCommand._configc C s� t |t�r t|�dkr |d n|}d}|r�|dkrBtd��|�}n�|dkrZtd��|�}nv|dv rrtd��|�}n^|d kr�td ��|�}nF|d v r�td ��|�}n.|d kr�td��|�}n|dv r�td��|�}|s�| jj�� td�}td| � dS )z�Output help information. :param args: the command to output help information about. If *args* is an empty, general help will be output. r Nr z�{} arg [value] arg: debuglevel, errorlevel, obsoletes, gpgcheck, assumeyes, exclude, repo_id.gpgcheck, repo_id.exclude If no value is given it prints the current value. If value is given it sets that value.r z{} [command] print help)r r z�{} arg [option] list: lists repositories and their status. option = [all | id | glob] enable: enable repositories. option = repository id disable: disable repositories. option = repository idr z"{} resolve the transaction set)r r zy{} arg list: lists the contents of the transaction reset: reset (zero-out) the transaction run: run the transactionr z{} run the transaction)r r z{} exit the shella� Shell specific arguments: config set config options help print help repository (or repo) enable, disable or list repositories resolvedep resolve the transaction set transaction (or ts) list, reset or run the transaction set run resolve and run the transaction set exit (or quit) exit the shellr> ) � isinstance�listr\ r r( r" rE � print_helprR )r! r` �arg�msgr r r rD � sF "������� zShellCommand._helpc C s� |r |d nd }|dv r6| � dd�|dd � � � n�|dv r�| jjj}d}|dd � D ]L}|�|�}|r|t||�� d}qXt�t d �d t d � | jj j � |�� qX|r�| j� � d | j_n | �d � d S ) Nr �rc Nz repolist rA r? )�enable�disableFTr@ zUnknown repo: '%s'r )rQ �joinr" r: r^ r_ r1 rM Zcriticalr �output�term�boldr< Z_compsrD )r! r` r r^ r< r �rr r r �_repo� s$ � zShellCommand._repoc C sN z| j j�| j jj� W n0 tjjyH } zt|� W Y d }~n d }~0 0 d S r6 ) r" r: r r. Z allow_erasingr rK Z DepsolveErrorrR )r! r` rP r r r �_resolve� s zShellCommand._resolvec C s� zTt |d��6}|�� }|D ]}|�d�s| �|� qW d � n1 sH0 Y W n8 ty� t�td�| jj j � |�� t � d� Y n0 d S )Nrn �#z!Error: Cannot open %s for readingr? )�open� readlinesr0 rQ �IOErrorrM �infor r: rk rl rm �sysr )r! �file�fd�linesrO r r r r8 � s . zShellCommand._run_scriptc C s� |r |d nd }|dkr$| � � d S | �� |dv rZ| jjr�| jj�| jj�}t�|� n||dkr�z| j�� W nB t j j y� } z&t� t d�d t|� � W Y d }~nd }~0 0 t�t d�� | � � n | �d� d S ) Nr �resetrg r r@ rA z Complete!r )r= rp r: r; rk Zlist_transactionrM ru Zdo_transactionr rK rL rN r r rD )r! r` r �outrP r r r r; s" 0 zShellCommand._transactionc C s | � dg� d S )Nr )r; �r! r` r r r �_ts_run" s zShellCommand._ts_runc C s t �td�� t�d� d S )Nz Leaving Shellr )rM ru r rv r r| r r r �_quit% s zShellCommand._quit)N)N)N)N)N)N)N)r r r �aliasesr r( r r) r* ZsummaryrF r � staticmethodr, r5 r r= rQ ra rD ro rp r8 r; r} r~ r r r r r , s8 � & ; r )Zdnf.clir Zdnf.i18nr r Zdnf.utilr r rI ZloggingrB rv Z getLoggerrM �objectr r r r r r r r �<module> s __pycache__/install.cpython-39.pyc 0000644 00000014005 15025176534 0013063 0 ustar 00 a �!cS � @ s� d dl mZ d dl mZ d dlZd dlmZ d dlZd dlZd dl m Z d dl m Z d dl mZ e�d�ZG d d � d e j�ZdS ) � )�absolute_import)�unicode_literalsN)�chain)�commands)� OptionParser)�_�dnfc @ s� e Zd ZdZejejejd�ZdZ de e� � � Z e d�Zedd� �Zdd � Zd d � Zd d � Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� ZdS )�InstallCommandzUA class containing methods needed by the cli to execute the install command. )z install-nz install-naz install-nevrazalternative-for({}))�install� localinstall�inz,install a package or packages on your systemc C s" | j ddtd�tjtd�d� d S )N�package�+ZPACKAGEzPackage to install)�nargs�metavar�action�help)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �</usr/lib/python3.9/site-packages/dnf/cli/commands/install.py� set_argparser1 s �z InstallCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� dS )z�Verify that conditions are met so that this command can run. That there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. TN) �cli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure7 s zInstallCommand.configurec C sL g }g }g }| � � }| j�| j� | jjdkrf| jjs>| jjrf| �| jj� | jj j rft j � td���g }| jj�rP| jjdk�rPt jj�rHz,t jj�| j�}|j| jj| jj j d� W n� t j j�yD } zn|jr�|jD ]}|�|� q�|j�r |jD ]}|�d| � q�|j} | �r0t�t jj�| d �� W Y d }~n d }~0 0 n| jj}| jj �r�|�r�| � | jj � | jj j �r�t j � td���n| � � }|�r�|�r�| � |� | jj j �r�t j � td���n |�r�| jjdk�r�| � |� | jjdk�r�| � |�}t!|�dk�s t!|�dk�s |�rH| jj j �rHt j j"td�d�#|�|d��d S ) Nr zNothing to do.��strict�@r zUnable to find a match� )�pkg_specZpackages)$� _get_nevra_forms_from_commandr Z _populate_update_security_filterr �command� grp_specs� pkg_specs� _log_not_valid_rpm_file_pathsr �confr r � exceptions�Errorr Z WITH_MODULES�module� module_baseZ ModuleBaser Z MarkingErrorsZno_match_group_specs�appendZerror_group_specs�module_depsolv_errors�logger�errorZ format_modular_solver_errorsr � _inform_not_a_valid_combination�_install_files�_install_groups�_install_packages�lenZPackagesNotAvailableError�join) r �err_pkgs�errsZerror_module_specs� nevra_formsZskipped_grp_specsr- �eZe_specr/ r r r �runE s\ � . �zInstallCommand.runc C s& | j j| jv r | j| j j gS g S d S )N)r r% r: )r r r r r$ z s z,InstallCommand._get_nevra_forms_from_commandc C sF t dd� |�}t| jj|�D ]$}td�}t�|| jjj � |�� q d S )Nc S s d| S )Nr! r )�gr r r �<lambda>� � z>InstallCommand._log_not_valid_rpm_file_paths.<locals>.<lambda>z Not a valid rpm file path: %s) �mapr r r' r r0 �infor �output�term�bold)r r& Z group_names�pkg�msgr r r r( � s z,InstallCommand._log_not_valid_rpm_file_pathsc C s. |D ]$}t d�}t�|| jjj�|�� qd S )NzNot a valid form: %s)r r0 Zwarningr rB rC rD )r �formsZformrF r r r r2 � s z.InstallCommand._inform_not_a_valid_combinationc C s� g }| j jj}| j j| jj|| j jjd�D ]\}z| j j||d� W q* t j j y� t d�}t �|| j jj�|j�� |�|� Y q*0 q*|S )N)r �progressr zNo match for argument: %s)r r) r Zadd_remote_rpmsr r rB rH Zpackage_installr r* � MarkingErrorr r0 rA rC rD �locationr. )r r8 r rE rF r r r r3 � s � z InstallCommand._install_filesc C sN z&| j j|t| j jj�| j jjd� W n" tjjyH | j jjrD� Y n0 d S )Nr ) r Zenv_group_install�tupler) Zgroup_package_typesr r r* r+ )r r& r r r r4 � s � z InstallCommand._install_groupsc C sV | j j�� j| j�|�d�}|rRtd�}t�|�|d� t t dd� |D ������ d S )N)Zprovidesz/There are following alternatives for "{0}": {1}z, c S s g | ] }|j �qS r )�name)�.0Zaltr r r � <listcomp>� r? z7InstallCommand._report_alternatives.<locals>.<listcomp>) r Zsack�queryZfilterm�alternatives_provide�formatr r0 rA r7 �sorted�set)r r# rO rF r r r �_report_alternatives� s ��z#InstallCommand._report_alternativesc C s� g }| j jj}| jjD ]�}z| j j|||d� W q tjjy� } zPd� |j | j j j � |��}t�|� | j �|� | �|� |�|� W Y d }~qd }~0 0 q|S )N)r rG z{}: {})r r) r r r' r r r* rI rQ �valuerB rC rD r0 rA Z_report_icase_hintrT r. )r r: r9 r r# r; rF r r r r5 � s "z InstallCommand._install_packagesN)�__name__� __module__� __qualname__�__doc__�hawkeyZ FORM_NAMEZFORM_NAZ FORM_NEVRAr: rP rK �keys�aliasesr Zsummary� staticmethodr r r< r$ r( r2 r3 r4 rT r5 r r r r r % s& � 5 r )Z __future__r r Zlogging� itertoolsr rZ Zdnf.exceptionsr Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr Z getLoggerr0 ZCommandr r r r r �<module> s __pycache__/repolist.cpython-39.opt-1.pyc 0000644 00000016517 15025176534 0014227 0 ustar 00 a �!cz2 � @ s� d dl mZ d dl mZ d dlmZ d dlmZmZmZm Z d dl m Z d dl Z d dlZ d dlZ d dlZd dlZd dlZd dlZe�d�Zdd � Zd d � Zd d � Zdd� ZG dd� dej�ZdS )� )�absolute_import)�unicode_literals)�commands)�_�ucd�fill_exact_width� exact_width)� OptionParserN�dnfc C sd |rt j�| j�� �ntd�}| jdkr4td�| S | jsFtd�| S t| j�}td�||f S d S )N�unknown���zNever (last: %s)zInstant (last: %s)z%s second(s) (last: %s))r �util�normalize_time�_repo� getTimestampr Zmetadata_expire� _num2ui_num)�repo�mdZlast�num� r �=/usr/lib/python3.9/site-packages/dnf/cli/commands/repolist.py� _expire_str% s r c C s t tj�d| d��S )Nz%dT)r r Zpycomp�format)r r r r r 0 s r c C sF | j �� }| j�� }|D ](}t�||�r. dS t�||�r dS qdS )NTF)�id�lower�name�fnmatch)r �patterns�ridZrnm�patr r r � _repo_match4 s r c C s: d}| j tjd�j|jd�D ]}||j7 }q tjj� |�S )Nr ��flags�Z reponame__eq) �query�hawkey�IGNORE_EXCLUDES�filtermr �_sizer �clir Z format_number)�sackr �ret�pkgr r r � _repo_size? s r- c @ s@ e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d S ) �RepoListCommandzVA class containing methods needed by the cli to execute the repolist command. )Zrepolist�repoinfoz,display the configured software repositoriesc C sx | � � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � | jddddg d�tjtd�d� d S )Nz--all� _repos_action� store_const�allzshow all repos)�dest�action�const�default�helpz --enabled�enabledz show enabled repos (default))r3 r4 r5 r7 z --disabled�disabledzshow disabled repos�repos�*�enabled-defaultZ REPOSITORY)r2 r8 r9 zRepository specification)�nargsr6 �metavar�choicesr4 r7 )� add_mutually_exclusive_group� add_argumentr r ZPkgNarrowCallback)�parserZ repolimitr r r � set_argparserN s$ ��� �z RepoListCommand.set_argparserc C s | j js | jjtjtjd� d S )N)�stdout�stderr)�opts�quietr) Zredirect_logger�loggingZWARNING�INFO)�selfr r r � pre_configure_ s z RepoListCommand.pre_configurec C sT | j js| j�� | jj}| jjjs0| j jdkr<d|_ d|_ | j j rP| j j | j _ d S )Nr/ T) rF rG r) Zredirect_repo_progress�demands�base�conf�verbose�commandZavailable_reposZsack_activationr0 � repos_action)rJ rL r r r � configurec s zRepoListCommand.configurec - C sR | j j}dd� | j jD �}| jjj}t| jj�� �}|jt � d�d� | j j }|j d |jd }|j d }|jd }d } g } |s�t�td �� d S |d kp�|d ko�|} g } |D �]�} t|�r�t| |�s�q�d \}}}d}d }d}| r�||| }}}| j�rfd}|dk�r q�| �s&|�s&| j jdk�rB|td� | }ttd��}|�sV| j jdk�r�t| jj| �}n:d}|dks�|d k�r�|�s�q�|td� | }ttd��}|�s�| j jdk�s�t| j�}| �|| j ||ff� q�|�r�| j }nd }| j � td�| j�| j � td�| j �g}| �r.|| j � td�|�g7 }|�r^| j � � �r^|| j � td�| j � � �g7 }|�r�| j �!� �r�| j �!� }|| j � td�d�"t#|���g7 }|�r | j �$� �r dd� | j �$� D �}|�%� D ]4\}}|| j � td �d |d�"t#|��f �g7 }�q�|�r�t| jjj&t'j(d �j)| jd ��} t| jj�&� j)| jd ��} t*| �} t*| �} | | 7 } || j � td �t+j,�-| j �.� ��| j � td!�| �| j � td"�| �| j � td#�|�g7 }| j/�r|| j � td$�| j/�g7 }|�r$| j �0� } || j � td%�t+j,�-| ��g7 }n"| j1�r$|| j � td&�| j1�g7 }| j2}!|!�rP|| j � td'�d�"|!��g7 }nF|�r�| j �3� }"|"�r�d(|"d t|"�d) f }#|| j � td'�|#�g7 }t4| |�}$|| j � td*�|$�g7 }| j5�r�|| j � td+�d�"| j5��g7 }| j6�r|| j � td,�d�"| j6��g7 }|�r&|| j � td-�|�g7 }| j7�rH|| j � td.�| j7�g7 }| �d/�"t8t|��� q�| �rtt9d0�"| �� |�s | �r ttd1��}%d }&d }'| D ]J\}}(\}}|%t|�k �r�t|�}%|&t|(�k �r�t|(�}&|'|k �r�|}'�q�| �r,ttd2��|'k�r|j:|%ttd2�� d3 })n|j:|%|' d3 })n|j:|%d) })|)|&k �rJ|)}&n$|)|&8 })|%|)d3 7 }%|&|)|)d3 7 }&t;td1�|%�}*| �r�t;td4�|&|&�}+ntd4�}+| �s�t9d5|*|+f � nt9d6|*|+td2�f � | D ]P\}}(\}}| �s�t9d5t;||%�|(f � �q�t9d6t;||%�t;|(|&|&�|f � �q�|�s4| j jdk�rNtd7�},t9|,�<t*| ��� d S )8Nc S s g | ] }|� � �qS r )r )�.0�xr r r � <listcomp>p � z'RepoListCommand.run.<locals>.<listcomp>r )�keyZgreenZboldZredZnormalr zNo repositories availabler2 r< )� rX rX rX Tr9 r/ r8 FzRepo-id : zRepo-name : zRepo-status : zRepo-revision : zRepo-tags : z, c S s i | ]\}}||�qS r r )rS �k�vr r r � <dictcomp>� rV z'RepoListCommand.run.<locals>.<dictcomp>zRepo-distro-tags : z[%s]: %sr! r# zRepo-updated : zRepo-pkgs : zRepo-available-pkgs: zRepo-size : zRepo-metalink : z Updated : zRepo-mirrors : zRepo-baseurl : z %s (%d more)� zRepo-expire : zRepo-exclude : zRepo-include : zRepo-excluded : zRepo-filename : � z zrepo id�status� z repo namez%s %sz%s %s %szTotal packages: {})=rF rQ r: rM rN rO �list�values�sort�operator� attrgetter�output�termZFG_COLORZMODE�loggerZwarningr �lenr r8 rP r r- r* r r �appendr �metadataZ fmtKeyValFillr Z getRevisionZgetContentTags�join�sortedZ getDistroTags�itemsr$ r% r& r' r r r r ZgetMaxTimestampZmetalinkr Z mirrorlistZbaseurlZ getMirrorsr Z excludepkgsZ includepkgsZrepofile�map�print�columnsr r )-rJ �argZextcmdsrO r: rf Z on_ehibegZ on_dhibegZon_hiendZtot_numZcolsZinclude_statusZrepoinfo_outputr ZehibegZdhibegZhiendZ ui_enabledZ ui_endis_widZui_excludes_numr8 Zui_sizer r �out�tagsZdistroTagsDictZdistror Z num_availableZui_numZui_num_available�tsZbaseurlsZmirrors�urlZexpireZid_lenZnm_lenZst_lenZrname�leftZtxt_ridZtxt_rnam�msgr r r �runn sP ��� �� ���� � �� � ��� � zRepoListCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodrC rK rR rx r r r r r. F s r. ) Z __future__r r Zdnf.clir Zdnf.i18nr r r r Zdnf.cli.option_parserr Zdnf.cli.formatr Z dnf.pycompZdnf.utilr r% rH rc Z getLoggerrg r r r r- ZCommandr. r r r r �<module> s" __pycache__/group.cpython-39.opt-1.pyc 0000644 00000024075 15025176534 0013520 0 ustar 00 a �!cq: � @ s� d dl mZ d dl mZ d dlmZ d dlmZ d dlmZm Z d dl Z d dlZ d dl Z d dlZ d dlZe�d�ZG dd � d ej�ZdS ) � )�absolute_import)�unicode_literals)� CompsQuery)�commands)�_�ucdN�dnfc s� e Zd ZdZddddddd�Zdee�� � Zed�Z d dd �Z d Z d Z d d� Z � fdd�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zed#d$� �Zd%d&� Zd'd(� Zd)d*� Z � Z S )+� GroupCommandz; Single sub-command interface for most groups interaction. �list�install�remove�info)Z grouplistZ groupinstallZ groupupdateZ groupremoveZ grouperaseZ groupinfo)�group�groups�grpz'display, or use, the groups information�upgrade)�updateZerase)r r )�summaryr r r r r �markc C sn | j �| jj�}|r<| jjd ur4| jj�d| jj� || j_| jjd u rPd| j_| j�| jj| jj�| j_d S )Nr r )�direct_commands�get�optsZcommand�subcmd�args�insert� _CMD_ALIASES)�selfZdirect� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/group.py� _canonical6 s �zGroupCommand._canonicalc s t t| ��|� d| _d S )NF)�superr �__init__�_remark)r �cli�� __class__r r r! C s zGroupCommand.__init__c C s$ t d�}t| jj�s tj�|��d S )Nz4No group data available for configured repositories.)r �len�base�compsr � exceptionsZ CompsError)r �msgr r r � _assert_compsG s zGroupCommand._assert_compsc sT � fdd�}� � � |d u r(� jjj}n� jj�d�|��}tj�t tj� ||��S )Nc s � j jj�| j�}| S �N)r' �history�envr �id)r. Z env_found�r r r �available_predM s z7GroupCommand._environment_lists.<locals>.available_pred�,) r+ r' r( � environments�environments_by_pattern�joinr �utilZmapallr � partition)r �patternsr1 �envsr r0 r �_environment_listsL s z GroupCommand._environment_listsc sz � fdd�}g }g }� � � |d u r0� jjj}n� jj�d�|��}|D ](}|}||�r\|}|rf|jrH|�|� qH||fS )Nc s � j jj�| j�}|rdS dS )NTF)r' r- r r r/ )r Z group_foundr0 r r �installed_predZ s z1GroupCommand._group_lists.<locals>.installed_predr2 )r+ r' r( r �groups_by_patternr5 � uservisible�append) r r= r8 r; � installed� availableZgrpsr Ztgt_listr r0 r � _group_listsY s zGroupCommand._group_listsc C sr |D ]d}d}| j j�|�D ]}| j�|� d}q| j j�|�D ]}| j�|� d}q>|st�t d�|� qdg fS )NFTz!Warning: Group %s does not exist.r ) r' r( r4 �outputZ display_groups_in_environmentr< Zdisplay_pkgs_in_groups�logger�errorr )r �userlistZstrngZ group_matchedr. r r r r �_infoq s zGroupCommand._infoc sh d}d}d}| j jjp | jj�|r�|d dkr>d}|�d� q |d dkrZd}|�d� q |d dkrvd}|�d� q |d dkr�d�|�d� q q�q | jjr�d}| jjr�d}| jjr�d}|s�d }d}|d u�r4|D ]P}| j j }t |� |��dk}t |� |��dk} |s�| s�t �td �d |� d}q�|�r4dg fS | �|�\} } | �||�\} } � �fd d �}�fd d�}|�s�|td�| � |�s�|td�| � |�s�d� | D ]$}|j�r��q�|td�|� d� �q�d� | D ]$}|j�s�q�|td�|� d� �q�|�rdg fS d� | D ]$}|j�r �q |td�|� d� �q d� | D ]$}|j�sJ�q:|td�|� d� �q:dg fS )N� r �hiddenr? r@ �idsTFzWarning: No groups match:z %sc sV � s t | � dtj�|j� }�r0|d|j 7 }|jrD|d|j 7 }t d�|�� d S )N� %s� (%s)z [%s]z{})�printr r6 �_name_unset_wrapper�ui_namer/ � lang_only�format)�sectr r* ��done� print_idsr r �_out_grp� s z$GroupCommand._list.<locals>._out_grpc sF |r t | � |D ]0}dtj�|j� }� r8|d|j 7 }t |� qd S )NrJ rK )rL r r6 rM rN r/ )rQ r9 �er* )rT r r �_out_env� s z$GroupCommand._list.<locals>._out_envz Available Environment Groups:z Installed Environment Groups:�Installed Groups:�Installed Language Groups:�Available Groups:�Available Language Groups:)r' �conf�verboser rI �poprH r? r@ r( r& r< r4 rC rD r r: rA rO )r rE r= Z showinstalledZ showavailableZerrsr r( Zin_groupZin_environmentZenv_instZ env_availr? r@ rU rW r rR r �_list� s� � zGroupCommand._listc C s� t | jj| jjt jt jB t jt jB �}| j�� }|j |� }| j j rXt | jj jdg �}nt | jj j�}tj�|�}|jD ]}|�||� qx|jD ]}|�||� q�d S )N�optional)r r' r( r- �GROUPS� ENVIRONMENTSZ AVAILABLE� INSTALLED�_build_comps_solverr r � with_optional�tupler\ �group_package_types�libdnfZ transactionZlistToCompsPackageTyper3 Z_environment_installr Z_group_install) r r8 �q�solver�res�typesZ pkg_types�env_idZgroup_idr r r � _mark_install� s � zGroupCommand._mark_installc C sd t | jj| jjt jt jB t j�}| j�� }|j|� }|j D ]}|� |� q:|j D ]}|� |� qPd S r, ) r r' r( r- ra rb rc rd r r3 Z_environment_remover Z _group_remove)r r8 ri rj rk rm Zgrp_idr r r � _mark_remove� s � zGroupCommand._mark_removec C s* |d | j v r"|d |dd � fS d|fS )Nr rG r )� _MARK_CMDS)r �extcmdsr r r � _mark_subcmd s zGroupCommand._mark_subcmdc C s d}t |�dkr*|d dkr*d}|�d� | jjr6d}|s>d }| �||�\}}dd� }d}|D ]}|jrjq^|d7 }q^|td�|� d}|D ]}|js�q�|d7 }q�|td�|� d}|D ]}|jr�q�|d7 }q�|td �|� d}|D ]}|js�q�|d7 }q�|td �|� dg fS ) NrG r rH c S s |sd S t �d| |� d S )Nz%s %u)rC r )rQ Znumr r r rU s z'GroupCommand._summary.<locals>._out_grprX rY FrZ r[ )r& r^ r rH rA rO r )r rE r= r? r@ rU rS r r r r �_summary sH zGroupCommand._summaryc C s� | j ddtd�d� | �� }|j ddtd�d� |j ddtd�d� |j d dtd �d� |j d dtd �d� | j d ddtd��tjd d�tjdd � ��d� | j dddtd�d� d S )Nz--with-optional� store_truez$include optional packages from group)�action�helpz--hiddenzshow also hidden groupsz --installedzshow only installed groupsz --availablezshow only available groupsz--idszshow also ID of groupsr �?ZCOMMANDz'available subcommands: {} (default), {}r �, rG )�nargs�metavarrv r �*Z COMMAND_ARGz argument for group subcommand)� add_argumentr � add_mutually_exclusive_grouprP r �_GROUP_SUBCOMMANDSr5 )�parserZ grpparserr r r � set_argparser< s2 ����� �� �zGroupCommand.set_argparserc C s� | � � | jj}| jj}|| jvrBt�td�d�| j�� t j j �|dv rd|sd| j j � | � t j j �| j j}d|_|dv r�d|_d|_|dkr�d|_d|_nd|_|dvr�t�| j� |dv r�t�| j| j � d S ) Nz$Invalid groups sub-command, use: %s.rx )r r r r T)r r r r r F)r r )r r r r r~ rC Zcriticalr r5 r r# ZCliErrorZ optparser� print_help�demandsZsack_activationZ root_userZ resolvingZ allow_erasingZavailable_reposr Z_checkEnabledRepor' Z _checkGPGKey)r �cmdr r� r r r � configureP s0 � zGroupCommand.configurec C s� | j j}| j j}|dkr"| �|�S |dkr4| �|�S |dkrF| �|�S |dkrx| �|�\}}|dkrn| �|�S | �|�S |dk�r(| j j r�t | j j j dg �}nt | j j j �}d| _z| j �||| j j j�W S tjj�y& } z<td �}t�|| j jj�|�� tj�td ���W Y d }~n d }~0 0 |d k�r>| j �|�S |dk�r�|D ]2}z| j � |g� W n tjj �yz Y n0 �qLd S ) Nr r r r r r r` TzNo package %s available.z)Unable to find a mandatory group package.r ) r r r rs r_ rF rr ro rn re rf r' r\ rg r" Zenv_group_install�strictr r) Z MarkingErrorr rC r rB ZtermZboldZPackagesNotAvailableErrorZenv_group_upgradeZenv_group_remove�Error)r r� rq r rl rV r* �argr r r �runo sH �� zGroupCommand.runc C sb | j s d S | jj}| jj}|j}| jj�� �� j|d�D ]$}|j � |�}|� ||� ||�� q8d S )N)�name) r" r' Z_goalr- Z group_membersZsack�queryr? ZfiltermZrpmZ get_reasonZ set_reasonZ group_reason)r Zgoalr- �names�pkg�reasonr r r �run_transaction� s z GroupCommand.run_transaction) �__name__� __module__� __qualname__�__doc__r rf �keys�aliasesr r r rp r~ r r! r+ r: rA rF r_ rn ro rr rs � staticmethodr� r� r� r� � __classcell__r r r$ r r $ s> �� h / *r )Z __future__r r Z dnf.compsr Zdnf.clir Zdnf.i18nr r Zlibdnf.transactionrh r Zdnf.exceptionsZdnf.utilZloggingZ getLoggerrC ZCommandr r r r r �<module> s __pycache__/mark.cpython-39.opt-1.pyc 0000644 00000005620 15025176534 0013311 0 ustar 00 a �!c� � @ sp d dl mZ d dl mZ d dlZd dlmZ d dlmZ d dl Z d dl Z d dl Z e � d�Z G dd� dej�ZdS ) � )�print_function)�unicode_literalsN)�_)�commands�dnfc @ sL e Zd ZdZed�Zedd� �Zdd� Zdd� Z d d � Z d d � Z d d� Z dS )� MarkCommand)�markz7mark or unmark installed packages as installed by user.c C s4 | j ddg d�td�d� | j dddtd �d � d S ) Nr � )�install�remove�groupzhinstall: mark as installed by user remove: unmark as installed by user group: mark as installed by group)�nargs�choices�help�package�+ZPACKAGEzPackage specification)r �metavarr )� add_argumentr )�parser� r �9/usr/lib/python3.9/site-packages/dnf/cli/commands/mark.py� set_argparser) s � �zMarkCommand.set_argparserc C s, | j j�|tjj� t�td�t |�� d S )Nz %s marked as user installed.) �base�history� set_reason�libdnf� transactionZTransactionItemReason_USER�logger�infor �str��self�pkgr r r � _mark_install2 s zMarkCommand._mark_installc C s, | j j�|tjj� t�td�t |�� d S )Nz %s unmarked as user installed.) r r r r r Z TransactionItemReason_DEPENDENCYr r r r r r r r � _mark_remove6 s zMarkCommand._mark_removec C s, | j j�|tjj� t�td�t |�� d S )Nz %s marked as group installed.) r r r r r ZTransactionItemReason_GROUPr r r r r r r r � _mark_group: s zMarkCommand._mark_groupc C s$ | j j}d|_d|_d|_d|_d S )NTF)�cli�demandsZsack_activationZ root_userZavailable_reposZ resolving)r! r' r r r � configure> s zMarkCommand.configurec C s� | j jd }| j j}t�t| d| ��}g }|D ]F}tj�|�}|� | j j �}|D ] }||� qRt |�dkr0|� |� q0|r�t�td�� |D ]}t�td�|� q�tjj�| j j�� }|d u r�| j j�� } n|j} | j j�| g g � | j j�| � d S )Nr Z_mark_zError:z Package %s is not installed.)Zoptsr r � functools�partial�getattrr ZsubjectZSubjectZget_best_queryr Zsack�len�appendr �errorr r& ZCliErrorr ZlastZ_tsZdbCookieZend_rpmdb_versionZbeg�end) r! �cmdZpkgsZ mark_funcZnotfoundr" Zsubj�q�oldZ rpmdb_versionr r r �runE s, zMarkCommand.runN) �__name__� __module__� __qualname__�aliasesr Zsummary� staticmethodr r# r$ r% r( r3 r r r r r $ s r )Z __future__r r Zlibdnf.transactionr Zdnf.i18nr Zdnf.clir r r) ZloggingZ getLoggerr ZCommandr r r r r �<module> s __pycache__/updateinfo.cpython-39.pyc 0000644 00000033276 15025176534 0013566 0 ustar 00 a �!c:J � @ s� d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlm Z ddl m Z ddl m Z mZ dd lmZ d d � ZG d d � d e j�ZdS )zUpdateInfo CLI command.� )�absolute_import)�print_function)�unicode_literalsN)�commands)� OptionParser)�_� exact_width��unicodec C s t dd� | D ��S )z7Return maximum length of items in a non-empty iterable.c s s | ]}t |�V qd S �N)r )�.0�item� r �?/usr/lib/python3.9/site-packages/dnf/cli/commands/updateinfo.py� <genexpr>& � z_maxlen.<locals>.<genexpr>)�max)�iterabler r r �_maxlen$ s r c s0 e Zd ZdZejed�ejed�ejed�ej ed�ej ed�iZ ed�ed�ed �ed �d �Z d d d d d d dd�Z ddgee �� � Zed�ZdZdddegZ� fdd�Zedd� �Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Z d'd(� Z d)d*� Z d+d,� Z d-d.� Z d/d0� Z!d1d2� Z"d3d4� Z#d5d6� Z$� Z%S )7�UpdateInfoCommandz)Implementation of the UpdateInfo command.�bugfix� enhancement�security�unknown� newpackagez Critical/Sec.zImportant/Sec.z Moderate/Sec.zLow/Sec.)�Critical� Important�Moderate�Low�list�info�summary)zlist-updateinfoz list-securityzlist-seczinfo-updateinfoz info-securityzinfo-seczsummary-updateinfoZ updateinfoZupifz!display advisories about packages� available� installed�updates�allc s t t| ��|� d| _dS )zInitialize the command.N)�superr �__init__�_installed_query)�self�cli�� __class__r r r' C s zUpdateInfoCommand.__init__c C s | � � }|jddddtd�d� |jddddtd �d� |jd dd dtd �d� |jd dddtd�d� g d�}| � � }|jddddtd�d� |jddddtd�d� |jddddtd�d� | jdd d d td �d � | jd!d"d d td#�d � | jd$d%d&||d' tjtd(�d)� d S )*Nz --available� _availabilityr" � store_constz?advisories about newer versions of installed packages (default))�dest�const�action�helpz --installedr# z?advisories about equal and older versions of installed packagesz --updatesr$ zbadvisories about newer versions of those installed packages for which a newer version is availablez--allr% z3advisories about any versions of installed packages)r! r r z --summary� _spec_actionr! z$show summary of advisories (default)z--listr zshow list of advisoriesz--infor zshow info of advisoriesz --with-cve�with_cveF� store_truez'show only advisories with CVE reference)r/ �defaultr1 r2 z --with-bz�with_bzz,show only advisories with bugzilla reference�spec�*ZSPECr zPackage specification)�nargs�metavar�choicesr6 r1 r2 )� add_mutually_exclusive_group� add_argumentr r ZPkgNarrowCallback)�parser� availabilityZcmdsZ output_formatr r r � set_argparserH sX ���� � � � � � �z UpdateInfoCommand.set_argparserc C s� d| j j_d| j j_| jj| jv r6| j| jj | j_n| jjrJ| jj| j_| jj r`| jj | j_ n8| jj rz| jj d | j vr�| j | j_ n| jj �d�| j_ t� | j_| jjr�| jj�tj� | jjr�| jj�tj� | jjr�| jj�tj� | jj�r| jj�tj� | jj �r�| jj �d�}|dk�r8| jj�tj� n�|dk�rT| jj�tj� np|dv �rp| jj�tj� nT|dk�r�| jj�tj� n8|dv �r�d| j_n$|dk�r�d| j_ n| jj � d|� | jj �r�| jj � | jj � d S ) zADo any command-specific configuration based on command arguments.Tr r r )r Zsecr )Z bugzillasZbzs�cvesN) r* ZdemandsZavailable_reposZsack_activation�optsZcommand�direct_commands� spec_actionr3 r- r@ r8 �availabilities�availability_default�pop�set�_advisory_typesr �add�hawkey�ADVISORY_BUGFIXr �ADVISORY_ENHANCEMENTr �ADVISORY_NEWPACKAGEr �ADVISORY_SECURITYr7 r4 �insert�advisory�extend)r) r8 r r r � configuren sJ zUpdateInfoCommand.configurec C s� | j jdkr$| �| j j�}td�}n^| j jdkrH| �| j j�}td�}n:| j jdkrl| �| j j�}td�}n| �| j j�}td�}| j jdkr�| � |� n$| j jdkr�| � |� n | � ||� dS )z#Execute the command with arguments.r# r$ r% r" r r N) rC r@ �installed_apkg_adv_instsr8 r �updating_apkg_adv_insts�all_apkg_adv_insts�available_apkg_adv_instsrE � display_list� display_info�display_summary)r) �apkg_adv_insts� descriptionr r r �run� s zUpdateInfoCommand.runc C s@ | j d u r | jj�� �� �� | _ | j j|j|jd�}t |�dkS )N)�nameZevr__gter ) r( �base�sack�queryr# Zapply�filterr_ �evr�len)r) �apackage�qr r r �_newer_equal_installed� s z(UpdateInfoCommand._newer_equal_installedc s | j js<| j js<| j js<| j js<| j js<| j js<| j js<dS � j| j jv rNdS t � fdd�| j jD ��rldS | j jr�� j| j jv r�dS | j jr�t � fdd�| j jD ��r�dS | j jr�t � fdd�| j jD ��r�dS | j jr�t dd� � j D ��r�dS | j j�rt dd� � j D ���rdS d S ) NTc 3 s | ]}t �� j|�V qd S r )�fnmatch� fnmatchcase�id�r �pat�rR r r r � r z6UpdateInfoCommand._advisory_matcher.<locals>.<genexpr>c s g | ]}� � |��qS r )Z match_bug)r Zbugrn r r � <listcomp>� r z7UpdateInfoCommand._advisory_matcher.<locals>.<listcomp>c s g | ]}� � |��qS r )Z match_cve)r Zcvern r r ro � r c S s g | ]}|j tjk�qS r )�typerL � REFERENCE_CVE�r �refr r r ro � r c S s g | ]}|j tjk�qS r )rp rL �REFERENCE_BUGZILLArr r r r ro � r F) rC rJ r8 �severityZbugzillarB r4 r7 rp �any� references)r) rR r rn r �_advisory_matcher� s> ������"" z#UpdateInfoCommand._advisory_matcherc # sd |� |�D ]T� � �| jj�}| �|�}t� fdd�| jjD ��}|sH|r | �� �}� ||fV q dS )z4Return (adv. package, advisory, installed) triplets.c 3 s | ]}t �� j|�V qd S r )ri rj r_ rl �rf r r r � s �zAUpdateInfoCommand._apackage_advisory_installed.<locals>.<genexpr>N) Zget_advisory_pkgsZ get_advisoryr` ra rx rv rC r8 rh )r) Z pkgs_queryZcmptype�specsrR Zadvisory_matchZapackage_matchr# r ry r � _apackage_advisory_installed� s � z.UpdateInfoCommand._apackage_advisory_installedc C s@ | j j}|�� jdd�}|�� }|r<|�|�� j|jd��}|S )z<Return query containing packages of currently running kernelT)�empty)� sourcerpm)r` ra rb �filtermZget_running_kernel�unionr} )r) ra rg Zkernelr r r �running_kernel_pkgs� s z%UpdateInfoCommand.running_kernel_pkgsc C s8 | j j�� �� �d�}|�| �� �� �}| �|tj |�S )z5Return available (adv. package, adv., inst.) triplets� ) r` ra rb r# Zlatestr r� r{ rL �GT)r) rz rg r r r rX � s z*UpdateInfoCommand.available_apkg_adv_instsc C s" | � | jj�� �� tjtjB |�S �z5Return installed (adv. package, adv., inst.) triplets)r{ r` ra rb r# rL �LT�EQ�r) rz r r r rU � s �z*UpdateInfoCommand.installed_apkg_adv_instsc C s | � | jj�� jdd�tj|�S )z4Return updating (adv. package, adv., inst.) tripletsT)Z upgradable)r{ r` ra rb r~ rL r� r� r r r rV � s �z)UpdateInfoCommand.updating_apkg_adv_instsc C s( | � | jj�� �� tjtjB tjB |�S r� ) r{ r` ra rb r# rL r� r� r� r� r r r rW s �z$UpdateInfoCommand.all_apkg_adv_instsc C sR i }|D ]:\}}}|j ||j< |j tjkr|j |jf||j|jf< qt�|�� �S )z Make the summary of advisories.)rp rk rL rP ru � collections�Counter�values)r) r\ �id2type�apkgrR r# r r r �_summary s zUpdateInfoCommand._summaryc C sR | � |�}|�r8ttd�| � dtd�|tj fdtd�|tj fdtd�|tjdf fdtd�|tjd f fdtd �|tjd f fdtd �|tjd f fdtd�|tjdf fdtd�|tj fdtd�|tj fdtd�|tj fg }t dd� |D ��}|D ]4\}}}|�s�qtd|d| t |�|f � �q| j j j �rN| j�� dS )z"Display the summary of advisories.z Updates Information Summary: r zNew Package notice(s)zSecurity notice(s)r� zCritical Security notice(s)r z Important Security notice(s)r zModerate Security notice(s)r zLow Security notice(s)r zUnknown Security notice(s)NzBugfix notice(s)zEnhancement notice(s)zother notice(s)c s s" | ]}|d rt |d �V qdS )� Nr )r �vr r r r ( r z4UpdateInfoCommand.display_summary.<locals>.<genexpr>z %*s %s� )r� �printr rL rO rP rM rN �ADVISORY_UNKNOWNr r r` �confZautocheck_running_kernelr* Z_check_running_kernel) r) r\ r] Ztyp2cntZ label_counts�width�indent�label�countr r r r[ s> � � � � ��" z!UpdateInfoCommand.display_summaryc s� � fdd�}� fdd�}t � }|D ]�\}}}d|j|j|jf }� jjsP� jjr�|jD ]V} | jt j krr� jjsrqVn| jt j kr�� jjs�qV|j|j f|� |||jft � �| j< qVq"|j|j f|� |||jft � �|j< q"g } d } } } t|�� dd� d �D ]l\\}}}}t| t|��} |�� D ]F\}}t| t|��} ||� }t| t|��} | �||�||||f� �q q�| D ]R\}}}}}� jjj�r�td || || || ||f � ntd || || ||f � �qld S ) z Display the list of advisories.c s � j jdksdS | rdS dS d S )Nr% � zi � )rC r@ )�inst�r) r r � inst2mark2 s z1UpdateInfoCommand.display_list.<locals>.inst2markc s2 | t jkr � j�|td��S � j�| td��S d S )Nz Unknown/Sec.r )rL rP �SECURITY2LABEL�getr � TYPE2LABEL)�typZsevr� r r � type2label: s z2UpdateInfoCommand.display_list.<locals>.type2labelz%s-%s.%sr c S s | d S )Nr r ��xr r r �<lambda>R r z0UpdateInfoCommand.display_list.<locals>.<lambda>��keyz%s%-*s %-*s %-*s %sz%s%-*s %-*s %sN)�dictr_ rd �archrC r4 r7 rw rp rL rt rq ru � setdefault�updatedrk �sorted�itemsr re �appendr` r� �verboser� )r) r\ r� r� Znevra_inst_dictr� rR r# Znevrars ZadvlistZidwZtlwZnwr� Zaupdatedr� ZaidZatypesevr� r r� r rY 0 s8 � � " z UpdateInfoCommand.display_listc s� �j j�� � �j jj�td�td�td�td�td�td�td�td�td �td �f �� ���fd d �}t� }|D ]\}}}|�|||�� qrtd � t |dd� d��� dS )z/Display the details about available advisories.z Update ID�TypeZUpdatedZBugsZCVEsZ DescriptionZSeverityZRightsZFilesZ Installedc s� | j g�j�| jtd��gt| j�gg g | jp0d�� | j g| j pBd�� t t � fdd�| j D ���d g }| jD ]L}|jtjkr�|d �d�|j |jp�d�� ql|jtjkrl|d �|j � ql|d �� |d �� �s�d |d< d |d < �jjd k�r|�rtd �ntd �g|d < t��}g }|�d� |�d| j � |�d� t�|�D ]f\}}|d d gfv �rf�qJt|�D ]>\}} |dk�r�|nd} |t| � } |�d| d| | f � �qn�qJd� |�S )Nr r� c 3 s | ]}|j � v r|jV qd S r )r� �filename)r �pkg)�archesr r r s s �zHUpdateInfoCommand.display_info.<locals>.advisory2info.<locals>.<genexpr>� z{} - {}r� � � r% �trueZfalse� zO===============================================================================r� r z %*s%s: %s� ) rk r� r� rp r r r� r] � splitlinesru Zrightsr� rI Zpackagesrw rL rt r� �format�titlerq �sortrC r@ r �zip� enumerater �join) rR r# Z attributesrs r� �linesr� Z atr_lines�i�liner� Z key_padding�r� �labelsr) r� r r � advisory2infoi sH � z5UpdateInfoCommand.display_info.<locals>.advisory2infoz c S s | � � S r )�lowerr� r r r r� � r z0UpdateInfoCommand.display_info.<locals>.<lambda>r� N) r` ra Z list_archesr� r� r rI rK r� r� r� )r) r\ r� Z advisoriesr� rR r# r r� r rZ a s �(z UpdateInfoCommand.display_info)&�__name__� __module__� __qualname__�__doc__rL rM r rN rP r� rO r� r� rD r �keys�aliasesr! rG rF r' � staticmethodrA rT r^ rh rx r{ r� rX rU rV rW r� r[ rY rZ � __classcell__r r r+ r r ) sR ��� %6 1r )r� Z __future__r r r r� ri rL Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr r Z dnf.pycompr r ZCommandr r r r r �<module> s __pycache__/check.cpython-39.opt-1.pyc 0000644 00000007261 15025176534 0013437 0 ustar 00 a �!c? � @ sV d dl mZ d dl mZ d dlmZ d dlmZ d dlZd dlZ G dd� dej �Z dS )� )�absolute_import)�unicode_literals)�_)�commandsNc @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) � CheckCommandzSA class containing methods needed by the cli to execute the check command. )�checkz#check for problems in the packagedbc C s� | j ddddtd�d� | j ddddtd �d� | j d ddd td �d� | j d dddtd�d� | j ddddtd�d� | j ddddd ddg gtjd� d S )Nz--all� check_types� append_const�allzshow all problems; default)�dest�action�const�helpz--dependencies� dependencieszshow dependency problemsz --duplicates� duplicateszshow duplicate problemsz --obsoleted� obsoletedzshow obsoleted packagesz --provides�provideszshow problems with provides�check_yum_types�*)�nargs�choicesr )� add_argumentr �argparse�SUPPRESS)�parser� r �:/usr/lib/python3.9/site-packages/dnf/cli/commands/check.py� set_argparser$ s2 ����� ��zCheckCommand.set_argparserc C sx d| j j_| jjr<| jjr0| jj| jj | j_n | jj| j_| jjsPdh| j_nt| jj�| j_| jj j dg7 _ d S )NTr ) ZcliZdemandsZsack_activation�optsr r �set�base�confZdisable_excludes)�selfr r r � configure; s � zCheckCommand.configurec C sj t � }| jj�� �� }| jj�ddh��r�d }|D �]f}t |j�t t |j �t |j � �B D ]�}t |�� d�rnqZt |j|gd��sZt |�� d�r�|d u r�tj�| j�}tj�|�}|j t |�d� tj�|�}| jjj|_|j|dd� |�� }|r�qZtd�} |�| �| jj j � |�| jj j � |��� qZ|j D ]j} |j| gt | �� � d d �} | D ]B} d } |�| �| jj j � |�| jj j � | �| jj j � | ��� �qR�q.q2| jj�dd h��r4| j�!|�} |�"� �#| ��$� }|�%� D ]\\}}|�&� |d d � D ]<}td��| jj j � |d �| jj j � |��} |�| � �q�q�| jj�ddh��r�|D ]p}|j'D ]b}|j|gt |�� � d d �}t |��rVtd��| jj j � |d �| jj j � |��} |�| � �qV�qL| jj�ddh��r2|D ]Z}|j(D ]L}||j|gd�v�r�td�} |�| �| jj j � |�| jj j � |��� �q�q�t)|�D ]} t*| � �q:|�rftj+�,d�t |����d S )Nr r Zrpmlib)r �(F)ZselectZoptionalz {} has missing requires of {}r )r �namez"{} has installed conflict "{}": {}r � z{} is a duplicate with {}r z{} is obsoleted by {}r z%{} provides {} but it cannot be foundz Check discovered {} problem(s))-r r �sack�queryZ installedr r � intersectionZregular_requiresZ requires_preZprereq_ignoreinst�str� startswith�len�filter�dnfZ rpmdb_sack�selectorZSelector�goalZGoalr! Zprotect_running_kernel�install�runr �add�format�outputZtermZbold� conflicts�splitZ_get_installonly_queryZ duplicated� differenceZ _name_dict�items�sortZ obsoletesr �sorted�print� exceptions�Error)r" Z output_set�qr' �pkgZrequirer/ r0 Zsolved�msgZconflictZ conflictedZ conflict_pkgZ installonlyZdupsr% Zpkgs�dupZobsoleter Zprovider r r r2 I s� & � �� � � � � �zCheckCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r# r2 r r r r r s r ) Z __future__r r Zdnf.i18nr Zdnf.clir r Zdnf.exceptionsr. ZCommandr r r r r �<module> s __pycache__/__init__.cpython-39.opt-1.pyc 0000644 00000060253 15025176534 0014121 0 ustar 00 a �!c�} � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZ ddl Z ddl Z ddl Z ddl Z ddlZe �d�Zed�d Zed �d Zed �Zd d� Zd dd�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG d d � d e�Z G d d � d e�Z dS )!z< Classes for subcommands of the yum command line interface. � )�print_function)�unicode_literals)� OptionParser)�_N�dnfz+To diagnose the problem, try running: '%s'.z rpm -Va --nofiles --nodigestzDYou probably have corrupted RPMDB, running '%s' might fix the issue.zrpm --rebuilddba You have enabled checking of packages via GPG keys. This is a good thing. However, you do not have any GPG public keys installed. You need to download the keys for packages you wish to install and install them. You can do that by running the command: rpm --import public.gpg.key Alternatively you can specify the url to the key you would like to use for a repository in the 'gpgkey' option in a repository section and {prog} will install it for you. For more information contact your distribution or package provider.c C sj | j js dS | �� sf| j�� D ]F}|js.|jr |js t�dt j t j j d�� t�td�|� t jj�q dS )z�Verify that there are gpg keys for the enabled repositories in the rpm database. :param base: a :class:`dnf.Base` object. :raises: :class:`cli.CliError` Nz %s ��progzProblem repository: %s)�confZgpgcheckZ_gpg_key_check�reposZ iter_enabledZ repo_gpgcheckZgpgkey�loggerZcritical�gpg_msg�formatr �util�MAIN_PROG_UPPERr �cli�CliError)�baser �repo� r �=/usr/lib/python3.9/site-packages/dnf/cli/commands/__init__.py� _checkGPGKey: s r r c C s| | j �� rdS |D ]@}|�d�r2tj�|�r2 dS tjj�|�d }|dv r dS qt d�� d� | j j ��}tj�|��dS )z�Verify that there is at least one enabled repo. :param base: a :class:`dnf.Base` object. :param possible_local_files: the list of strings that could be a local rpms :raises: :class:`cli.CliError`: Nz.rpmr )�http�ftp�file�httpsz*There are no enabled repositories in "{}".z", ")r Z _any_enabled�endswith�os�path�existsr Zpycomp�urlparser r �joinr Zreposdirr r )r Zpossible_local_filesZlfile�scheme�msgr r r �_checkEnabledRepoK s r# c @ s� e Zd ZdZg ZdZdZdd� Zedd� �Z edd � �Z ed d � �Z d d � Z dd� Z dd� Zdd� Zdd� Zdd� Zdd� ZdS )�Commandz%Abstract base class for CLI commands.� Nc C s || _ d S �N�r ��selfr r r r �__init__f s zCommand.__init__c C s | j jS r& )r r �r) r r r r j s z Command.basec C s | j d S )Nr ��aliasesr+ r r r �_basecmdo s zCommand._basecmdc C s | j jjS r& )r r �outputr+ r r r r/ s s zCommand.outputc C s dS )z4Define command specific options and arguments. #:apiNr )r) �parserr r r � set_argparserw s zCommand.set_argparserc C s dS )z*Do any command-specific pre-configuration.Nr r+ r r r � pre_configure{ s zCommand.pre_configurec C s dS )z&Do any command-specific configuration.Nr r+ r r r � configure� s zCommand.configurec C s& t |tjj�rttfS td| ��dS )z.Get suggestions for resolving the given error.zerror not supported yet: %sN)� isinstancer � exceptionsZTransactionCheckError� _RPM_VERIFY�_RPM_REBUILDDB�NotImplementedError)r) �errorr r r �get_error_output� s zCommand.get_error_outputc C s dS )zExecute the command.Nr r+ r r r �run� s z Command.runc C s dS )z$Finalize operation after resolvementNr r+ r r r � run_resolved� s zCommand.run_resolvedc C s dS )z%Finalize operations post-transaction.Nr r+ r r r �run_transaction� s zCommand.run_transaction)�__name__� __module__� __qualname__�__doc__r- �summary�optsr* �propertyr r. r/ r1 r2 r3 r: r; r< r= r r r r r$ _ s$ r$ c @ sR e Zd ZdZdZed�ZdZddddd d d d eh Ze d d� �Z dd� Z dd� Z dS )� InfoCommandzRA class containing methods needed by the cli to execute the info command. )�info�ifz4display details about a package or group of packages�all� available� installed�extras�updates�upgrades� autoremove�recent� obsoletesc C s� |� � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jdd td �| j| jtjtd �d � d S ) N�--all�_packages_action� store_constrH �show all packages (default)��dest�action�const�default�help� --availablerI � show only available packages�rV rW rX rZ � --installedrJ � show only installed packages�--extrasrK �show only extras packages� --updatesrM �show only upgrades packages� --upgrades� --autoremoverN � show only autoremove packages�--recentrO �#show only recently changed packages�packages�*�PACKAGEzPackage name specification��nargs�metavar�choicesrY rW rZ )� add_mutually_exclusive_group� add_argumentr � pkgnarrows�DEFAULT_PKGNARROWr �PkgNarrowCallback)�clsr0 �narrowsr r r r1 � sL ���������zInfoCommand.set_argparserc C s| | j j}d|_| jjr"| jj| j_| jjdkr4d|_| jjrd| jjr\| j �dd| jj � nd| j_| jjdkrxd| j_d S )NTrJ � --obsoletes�--rP rL rM ) r �demands�sack_activationrC rR �packages_action�available_reposrP �_option_conflict�r) ry r r r r3 � s zInfoCommand.configurec C s& | j �| j� | j�d| jj| jj�S )NrF �r � _populate_update_security_filterrC r �output_packagesr{ ri r+ r r r r; � s �zInfoCommand.runN) r> r? r@ rA r- r rB rs rr � classmethodr1 r3 r; r r r r rE � s � rE c @ s$ e Zd ZdZdZed�Zdd� ZdS )� ListCommandzRA class containing methods needed by the cli to execute the list command. )�listZlsz$list a package or groups of packagesc C s& | j �| j� | j�d| jj| jj�S )Nr� r r+ r r r r; � s �zListCommand.runN)r> r? r@ rA r- r rB r; r r r r r� � s r� c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �ProvidesCommandzVA class containing methods needed by the cli to execute the provides command. )�providesZ whatprovidesZprovZwpz*find what package provides the given valuec C s | j ddtd�td�d� d S )N� dependency�+ZPROVIDEz#Provide specification to search for�rm rn rZ �rq r �r0 r r r r1 � s �z ProvidesCommand.set_argparserc C s | j j}d|_d|_d|_d S �NTF)r ry r| Zfresh_metadatarz r~ r r r r3 � s zProvidesCommand.configurec C s t �td�� | j�| jj�S )NzSearching Packages: )r �debugr r r� rC r� r+ r r r r; � s zProvidesCommand.runN� r> r? r@ rA r- r rB � staticmethodr1 r3 r; r r r r r� � s r� c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �CheckUpdateCommandzZA class containing methods needed by the cli to execute the check-update command. )� check-updatez check-upgradez$check for available package upgradesc C s0 | j ddddtd�d� | j ddtd �d � d S ) Nz --changelogs� changelogsF� store_truez show changelogs before update)rV rY rW rZ ri rj rk )rm rn r� r� r r r r1 s �z CheckUpdateCommand.set_argparserc C s6 | j j}d|_d|_d|_| jjr(d|_t| j� d S �NT) r ry rz r| Zplugin_filtering_enabledrC r� r# r r~ r r r r3 s z CheckUpdateCommand.configurec C sR | j j| jdd� | jj| jjd| jjd�}|r:d| j j_| jj j rN| j � � d S )NZgte)Zcmp_typeT)�print_r� �d ) r r� rC r � check_updatesri r� ry �success_exit_statusr Zautocheck_running_kernelZ_check_running_kernel�r) �foundr r r r; s � zCheckUpdateCommand.runNr� r r r r r� � s r� c s e Zd ZdZG dd� de�ZG dd� de�ZG dd� de�ZG dd � d e�ZG d d � d e�Z G d d � d e�Z G dd� de�Z G dd� de�Z G dd� de�Z G dd� de�ZG dd� de�Zeeeee e e e e eeh ZdZed�Z� fdd�Zd d � Zd d � Zd d!� Z� ZS )"�RepoPkgsCommandz2Implementation of the repository-packages command.c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z%RepoPkgsCommand.CheckUpdateSubCommand�'Implementation of the info sub-command.)r� c C s | j j}d|_d|_d S r� )r ry r| rz r~ r r r r3 ( s z/RepoPkgsCommand.CheckUpdateSubCommand.configurec C s* | j j| jj| jdd�}|r&d| jj_dS )�?Execute the command with respect to given arguments *cli_args*.T)r� r� N)r r� rC � pkg_specs�reponamer ry r� r� r r r � run_on_repo- s �z1RepoPkgsCommand.CheckUpdateSubCommand.run_on_repoN�r> r? r@ rA r- r3 r� r r r r �CheckUpdateSubCommand# s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.InfoSubCommandr� )rF c C sh | j j}d|_| jjr"| jj| j_| jjdkr4d|_| jjrd| jjr\| j �dd| jj � nd| j_d S )NTrJ rw rx rP ) r ry rz rC �_pkg_specs_action�pkg_specs_actionr| rP r} r~ r r r r3 9 s z(RepoPkgsCommand.InfoSubCommand.configurec C s. | j �| j� | j�d| jj| jj| j� dS )r� rF N�r r� rC r r� r� r� r� r+ r r r r� F s �z*RepoPkgsCommand.InfoSubCommand.run_on_repoNr� r r r r �InfoSubCommand4 s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z!RepoPkgsCommand.InstallSubCommandz*Implementation of the install sub-command.)�installc C s$ | j j}d|_d|_d|_d|_d S r� �r ry r| rz � resolving� root_userr~ r r r r3 Q s z+RepoPkgsCommand.InstallSubCommand.configurec C s� | j �| j� t| j| j � d}| jjshz| j�d| j� W n$ tj j y` t � t d�� Y q�0 d}nt| jjD ]j}z| j�|| j� W nL tj j y� } z0d�|j| jjj�|��}t � |� W Y d }~qpd }~0 0 d}qp|s�tj �t d���d S )NFrj �No package available.Tz{}: {}z No packages marked for install.)r r� rC r r r� r� r� r r5 � MarkingErrorr rF r r �valuer/ �term�bold�Error)r) �done�pkg_spec�er" r r r r� X s$ z-RepoPkgsCommand.InstallSubCommand.run_on_repoNr� r r r r �InstallSubCommandL s r� c @ s e Zd ZdZdZdd� ZdS )z RepoPkgsCommand.ListSubCommandz'Implementation of the list sub-command.)r� c C s. | j �| j� | j�d| jj| jj| j� dS )r� r� Nr� r+ r r r r� z s �z*RepoPkgsCommand.ListSubCommand.run_on_repoN)r> r? r@ rA r- r� r r r r �ListSubCommandu s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.MoveToSubCommandz*Implementation of the move-to sub-command.)zmove-toc C s$ | j j}d|_d|_d|_d|_d S r� �r ry rz r| r� r� r~ r r r r3 � s z*RepoPkgsCommand.MoveToSubCommand.configurec C s� t | j| j� d}| jjs�z| jjd| jd� W nZ tjj yT t � t d�� Y n< tjj yv t � t d�� Y n tjjy� Y n0 d}n�| jjD ]�}z| jj|| jd� W n� tjj y� t d�}t � ||� Y q� tjj �yV } z\|jD ]H}d}| jj�|�}|�r t d �| }t d �}t � || jj�|�|� q�W Y d }~q�d }~0 tjj�yl Y q�0 d}q�|�s�tj�t d ���d S ) r� Frj )Z new_reponamezNo package installed.r� T�No match for argument: %sr% � (from %s)�%Installed package %s%s not available.N�Nothing to do.)r r r rC r� � reinstallr� r r5 �PackagesNotInstalledErrorr rF r �PackagesNotAvailableErrorr� ri �historyr r/ r� r� r� )r) r� r� r" �err�pkg�xmsg�pkgrepor r r r� � s> .z,RepoPkgsCommand.MoveToSubCommand.run_on_repoNr� r r r r �MoveToSubCommand� s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z&RepoPkgsCommand.ReinstallOldSubCommandz0Implementation of the reinstall-old sub-command.)z reinstall-oldc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 � s z0RepoPkgsCommand.ReinstallOldSubCommand.configurec C s� t | j| j� d}| jjs�z| j�d| j| j� W n^ tjj yZ t d�}t � |� Y n< tjj y| t � t d�� Y n tjjy� Y n0 d}n�| jjD ]�}z| j�|| j| j� W n� tjj y� t d�}t � ||� Y q� tjj �y` } z^|jD ]J}d}| jj�|�}|�r(t d�| }t d �}t � || jj�|�|� �q W Y d }~q�d }~0 tjj�yv Y q�0 d}q�|�s�tj�t d ���d S ) r� Frj �)No package installed from the repository.r� Tr� r% r� r� Nr� )r r r rC r� r� r� r r5 r� r r rF r� r� ri r� r r/ r� r� r� )r) r� r" r� r� r� r� r� r r r r� � sD � 0z2RepoPkgsCommand.ReinstallOldSubCommand.run_on_repoNr� r r r r �ReinstallOldSubCommand� s r� c s4 e Zd ZdZdZ� fdd�Zdd� Zdd� Z� ZS ) z#RepoPkgsCommand.ReinstallSubCommandz,Implementation of the reinstall sub-command.)r� c s, t tj| ��|� t�|�t�|�f| _dS )�Initialize the command.N)�superr� �ReinstallSubCommandr* r� r� �wrapped_commandsr( �� __class__r r r* � s �z,RepoPkgsCommand.ReinstallSubCommand.__init__c C s2 d| j j_| jD ] }| j|_| j|_|�� qd S r� )r ry r| r� rC r� r3 �r) Zcommandr r r r3 � s z-RepoPkgsCommand.ReinstallSubCommand.configurec C sZ t | j| j� | jD ]0}z |�� W n tjjy> Y qY q0 qVqtj�td���dS )r� z!No packages marked for reinstall.N) r r r r� r� r r5 r� r r� r r r r� s z/RepoPkgsCommand.ReinstallSubCommand.run_on_repo) r> r? r@ rA r- r* r3 r� � __classcell__r r r� r r� � s r� c @ s, e Zd ZdZdZdd� Zdd� Zdd� Zd S ) z,RepoPkgsCommand.RemoveOrDistroSyncSubCommandz8Implementation of the remove-or-distro-sync sub-command.)zremove-or-distro-syncc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 s z6RepoPkgsCommand.RemoveOrDistroSyncSubCommand.configurec s� | j jj��� tj�|�}|�| j jj�}| j jj� � �fdd�|� � D �}|s`tj � d|��|� � }| j jj j}|D ]<}|j|j|jd�r�| j jj�|� qx| j jjj||d� qxdS )z;Synchronize a package with another repository or remove it.c s g | ]}� � |��kr|�qS r )r )�.0r� �r� r� r r � <listcomp>" s �zIRepoPkgsCommand.RemoveOrDistroSyncSubCommand._replace.<locals>.<listcomp>zno package matched)�name�arch)� clean_depsN)r r ZsackZ disable_repor �subjectZSubjectZget_best_queryr� rJ r5 r� rI r Z clean_requirements_on_remove�filterr� r� Z_goalZ distupgradeZerase) r) r� r� r� �matchesrJ rI r� �packager r� r �_replace s" �� z5RepoPkgsCommand.RemoveOrDistroSyncSubCommand._replacec C s� t | j| j� d}| jjs\z| �d| j� W n( tjj yT t d�}t � |� Y q�0 d}nP| jjD ]F}z| �|| j� W n* tjj y� t d�}t � ||� Y qd0 d}qd|s�tj� t d���dS )r� Frj r� Tr� r� N)r r r rC r� r� r� r r5 r� r r rF r� �r) r� r" r� r r r r� 0 s$ z8RepoPkgsCommand.RemoveOrDistroSyncSubCommand.run_on_repoN)r> r? r@ rA r- r3 r� r� r r r r � RemoveOrDistroSyncSubCommand s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z+RepoPkgsCommand.RemoveOrReinstallSubCommandz6Implementation of the remove-or-reinstall sub-command.)zremove-or-reinstallc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 R s z5RepoPkgsCommand.RemoveOrReinstallSubCommand.configurec C s t | j| j� d}| jjszz | jjd| j| jdd� W n< tjj y^ t d�}t � |� Y q� tjj yr Y q�0 d}nn| jjD ]d}z | jj|| j| jdd� W n> tjj y� t d�}t � ||� Y q� tjj y� Y q�0 d}q�|s�tj�t d���dS ) r� Frj T)Z old_reponameZnew_reponame_neqZ remove_nar� r� r� N)r r r rC r� r� r� r r5 r� r r rF r� r� r� r r r r� Y s8 � � z7RepoPkgsCommand.RemoveOrReinstallSubCommand.run_on_repoNr� r r r r �RemoveOrReinstallSubCommandM s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z RepoPkgsCommand.RemoveSubCommandz)Implementation of the remove sub-command.)�removec C s* | j j}d|_d|_d|_d|_d|_d S r� )r ry rz Z allow_erasingr| r� r� r~ r r r r3 � s z*RepoPkgsCommand.RemoveSubCommand.configurec C s� d}| j jsPz| j�d| j� W n( tjjyH td�}t � |� Y q�0 d}n^| j jD ]T}z| j�|| j� W n6 tjjy� } zt � t |�� W Y d}~qXd}~0 0 d}qX|s�t � td�� dS )r� Frj r� TNz No packages marked for removal.) rC r� r r� r� r r5 r� r r rF �strZwarning)r) r� r" r� r� r r r r� � s $z,RepoPkgsCommand.RemoveSubCommand.run_on_repoNr� r r r r �RemoveSubCommand~ s r� c @ s$ e Zd ZdZdZdd� Zdd� ZdS )z!RepoPkgsCommand.UpgradeSubCommandz*Implementation of the upgrade sub-command.)�upgradez upgrade-toc C s$ | j j}d|_d|_d|_d|_d S r� r� r~ r r r r3 � s z+RepoPkgsCommand.UpgradeSubCommand.configurec C s� t | j| j� d}| jjs.| j�| j� d}nN| jjD ]D}z| j�|| j� W n& tj j yt t � t d�|� Y q60 d}q6|s�tj �t d���dS )r� FTr� z No packages marked for upgrade.N)r r r rC r� Z upgrade_allr� r� r r5 r� r rF r r� )r) r� r� r r r r� � s z-RepoPkgsCommand.UpgradeSubCommand.run_on_repoNr� r r r r �UpgradeSubCommand� s r� )zrepository-packagesz repo-pkgsz repo-packageszrepository-pkgsz7run commands on top of all packages in given repositoryc s> t t| ��� � � fdd�| jD �}d| _dd� |D �| _dS )r� c 3 s | ]}|� �V qd S r& r �r� �subcmdr' r r � <genexpr>� � z+RepoPkgsCommand.__init__.<locals>.<genexpr>Nc S s i | ]}|j D ] }||�qqS r r, �r� r� �aliasr r r � <dictcomp>� s z,RepoPkgsCommand.__init__.<locals>.<dictcomp>)r� r� r* �SUBCMDSr� �_subcmd_name2obj)r) r Z subcmd_objsr� r' r r* � s �zRepoPkgsCommand.__init__c C s` |� � }|jddddd td�d� |jddddtd �d � |jd ddd td �d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jddddtd�d � |jdd tjtd �td �d � d d!� | jD �}d"d!� | jD �}|jd#d d$|d%�|�d&� d}|d dddd'ddh}|jd(d)td*�||tjtd+�d,� d S )-NrQ r� rS rH rT rU r[ rI r\ r] r^ rJ r_ r` rK ra rb rM rc rd re rN rf rg rO rh r� � ZREPOIDz Repository ID)rm rW rn rZ c S s g | ]}|j d �qS )r r, r� r r r r� � r� z1RepoPkgsCommand.set_argparser.<locals>.<listcomp>c S s g | ]}|j D ]}|�qqS r r, r� r r r r� � r� r� Z SUBCOMMANDz, )rm rn ro rZ rP r� rj rk zPackage specificationrl )rp rq r r Z_RepoCallbackEnabler� r rt )r) r0 rv Zsubcommand_choicesZsubcommand_choices_allrs rr r r r r1 � sh �������� � ���z RepoPkgsCommand.set_argparserc C s� z| j | jjd | _W n@ tjjtfyX } z | jj�� tjj�W Y d}~n d}~0 0 | j| j_| jj d | j_ | j� � dS )z8Verify whether the command can run with given arguments.r N) r� rC r� r r r �KeyError� optparser� print_usager� r3 )r) r� r r r r3 s zRepoPkgsCommand.configurec C s | j �� dS )z>Execute the command with respect to given arguments *extcmds*.N)r� r� r+ r r r r; s zRepoPkgsCommand.run)r> r? r@ rA r$ r� r� r� r� r� r� r� r� r� r� r� r� r- r rB r* r1 r3 r; r� r r r� r r� s0 ) 79 >1(#� + r� c @ s0 e Zd ZdZdZed�Zedd� �Zdd� Z dS ) � HelpCommandzRA class containing methods needed by the cli to execute the help command. )rZ z display a helpful usage messagec C s* | j ddtd�td�jtjjd�d� d S )N�cmd�?ZCOMMANDz {prog} command to get help forr r� )rq r r r r r r� r r r r1 s ��zHelpCommand.set_argparserc C sL | j jr| j j| jjvr&| jj�� n"| jj| j j }| jj�|| �� d S r& )rC r� r Z cli_commandsr� � print_helpr� r r r r; $ s �zHelpCommand.runN) r> r? r@ rA r- r rB r� r1 r; r r r r r� s r� )r ) rA Z __future__r r Zdnf.cli.option_parserr Zdnf.i18nr Zdnf.clir Zdnf.exceptionsZ dnf.pycompZdnf.utilZloggingr Z getLoggerr r6 r7 r r r# �objectr$ rE r� r� r� r� r� r r r r �<module> s> ��� 9?$ y __pycache__/downgrade.cpython-39.pyc 0000644 00000003502 15025176534 0013367 0 ustar 00 a �!c � @ sR d dl mZ d dl mZ d dlmZ d dlmZ d dlmZ G dd� dej �Z dS ) � )�absolute_import)�unicode_literals)�commands)� OptionParser)�_c @ s8 e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d S ) �DowngradeCommandzWA class containing methods needed by the cli to execute the downgrade command. )Z downgradeZdgzDowngrade a packagec C s | j ddtd�tjd� d S )N�package�*zPackage to downgrade)�nargs�help�action)� add_argumentr r ZParseSpecGroupFileCallback)�parser� r �>/usr/lib/python3.9/site-packages/dnf/cli/commands/downgrade.py� set_argparser$ s �z DowngradeCommand.set_argparserc C sH | j j}d|_d|_d|_d|_t�| j| j � | j j sDt� | j� d S )NT) Zcli�demandsZsack_activationZavailable_reposZ resolvingZ root_userr Z _checkGPGKey�base�opts� filenamesZ_checkEnabledRepo)�selfr r r r � configure) s zDowngradeCommand.configurec C sJ | j j| jjd| j jjd�}| j j| jjdd� | jjD � || j j j d�S )NF)�strict�progressc S s g | ] }d | �qS )�@r )�.0�xr r r � <listcomp>8 � z(DowngradeCommand.run.<locals>.<listcomp>)Zspecs� file_pkgsr ) r Zadd_remote_rpmsr r �outputr Z downgradePkgsZ pkg_specsZ grp_specsZconfr )r r r r r �run4 s ��zDowngradeCommand.runN) �__name__� __module__� __qualname__�__doc__�aliasesr Zsummary� staticmethodr r r! r r r r r s r N) Z __future__r r Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr ZCommandr r r r r �<module> s __pycache__/mark.cpython-39.pyc 0000644 00000005620 15025176534 0012352 0 ustar 00 a �!c� � @ sp d dl mZ d dl mZ d dlZd dlmZ d dlmZ d dl Z d dl Z d dl Z e � d�Z G dd� dej�ZdS ) � )�print_function)�unicode_literalsN)�_)�commands�dnfc @ sL e Zd ZdZed�Zedd� �Zdd� Zdd� Z d d � Z d d � Z d d� Z dS )� MarkCommand)�markz7mark or unmark installed packages as installed by user.c C s4 | j ddg d�td�d� | j dddtd �d � d S ) Nr � )�install�remove�groupzhinstall: mark as installed by user remove: unmark as installed by user group: mark as installed by group)�nargs�choices�help�package�+ZPACKAGEzPackage specification)r �metavarr )� add_argumentr )�parser� r �9/usr/lib/python3.9/site-packages/dnf/cli/commands/mark.py� set_argparser) s � �zMarkCommand.set_argparserc C s, | j j�|tjj� t�td�t |�� d S )Nz %s marked as user installed.) �base�history� set_reason�libdnf� transactionZTransactionItemReason_USER�logger�infor �str��self�pkgr r r � _mark_install2 s zMarkCommand._mark_installc C s, | j j�|tjj� t�td�t |�� d S )Nz %s unmarked as user installed.) r r r r r Z TransactionItemReason_DEPENDENCYr r r r r r r r � _mark_remove6 s zMarkCommand._mark_removec C s, | j j�|tjj� t�td�t |�� d S )Nz %s marked as group installed.) r r r r r ZTransactionItemReason_GROUPr r r r r r r r � _mark_group: s zMarkCommand._mark_groupc C s$ | j j}d|_d|_d|_d|_d S )NTF)�cli�demandsZsack_activationZ root_userZavailable_reposZ resolving)r! r' r r r � configure> s zMarkCommand.configurec C s� | j jd }| j j}t�t| d| ��}g }|D ]F}tj�|�}|� | j j �}|D ] }||� qRt |�dkr0|� |� q0|r�t�td�� |D ]}t�td�|� q�tjj�| j j�� }|d u r�| j j�� } n|j} | j j�| g g � | j j�| � d S )Nr Z_mark_zError:z Package %s is not installed.)Zoptsr r � functools�partial�getattrr ZsubjectZSubjectZget_best_queryr Zsack�len�appendr �errorr r& ZCliErrorr ZlastZ_tsZdbCookieZend_rpmdb_versionZbeg�end) r! �cmdZpkgsZ mark_funcZnotfoundr" Zsubj�q�oldZ rpmdb_versionr r r �runE s, zMarkCommand.runN) �__name__� __module__� __qualname__�aliasesr Zsummary� staticmethodr r# r$ r% r( r3 r r r r r $ s r )Z __future__r r Zlibdnf.transactionr Zdnf.i18nr Zdnf.clir r r) ZloggingZ getLoggerr ZCommandr r r r r �<module> s __pycache__/search.cpython-39.pyc 0000644 00000010551 15025176534 0012664 0 ustar 00 a �!c� � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlmZ d dlmZ d dl m Z m Z m Z d dl Z d dlZ d dlZ d dlZd dlZe�d�ZG d d � d ej�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�commands)� OptionParser)�ucd�_�C_�dnfc @ sP e Zd ZdZdZed�Zedd� �Zdd� Z dd � Z d d � Z d d � Z dd� Z dS )� SearchCommandzTA class containing methods needed by the cli to execute the search command. )�searchZsez+search package details for the given stringc C s< | j ddtd�d� | j ddtd�dgd tjtd �d � d S ) Nz--all� store_truez'search also package description and URL)�action�help� query_string�+ZKEYWORD�allzKeyword to search for)�nargs�metavar�choices�defaultr r )� add_argumentr r ZPkgNarrowCallback)�parser� r �;/usr/lib/python3.9/site-packages/dnf/cli/commands/search.py� set_argparser0 s ��zSearchCommand.set_argparserc s$ t �dtdd�fdtdd�fdtdd�fdtd �ff�� � fd d ����fd d �}tj�� }|D ] }��|d|� ��|d|� q`�jj r�|D ] }��|d|� ��|d|� q�n6t |�}t |� � �}|D ] }t |� |��|kr�||= q�d}d} d} d} d} �jjj�s$�jj�� j|� � d��� } t� } |jd| d�D ]�}�jjj�sp|j|j | v �r^�q8| �|j|j � ||�|�k�r�|�|�}d} | |� |�k�r�|� |�} d} | |�|�| kk�r�|�|�| k} d} | �r�|| || � d} �jj � ||�|�|� �q8t |�dk�r t � td�� dS )z0Search for simple text tags in a package object.�nameZlong�Name�summaryZSummary� descriptionZ Description�urlZURLc s z � | W S | Y S 0 d S )Nr )�attr)� TRANS_TBLr r �_translate_attrC s z.SearchCommand._search.<locals>._translate_attrc s^ t � |�}td��|�}| r*td�| }n td�| }�jj�|d�|� �}tt|�� d S )Nz & z%s Exactly Matched: %%sz%s Matched: %%sz, )�mapr �join�base�outputZ fmtSection�printr )� exact_match�attrs�keysZ trans_attrsZtrans_attrs_strZ section_text� formatted)r# �selfr r �_print_section_headerI s z4SearchCommand._search.<locals>._print_section_headerNF)�pkgT)�reverseZlimit_tor zNo matches found.) � collections� OrderedDictr r r Z match_counterZ MatchCounter�_search_counted�optsr �len�listr+ �matched_needlesr& ZconfZshowdupesfromrepos�sack�query�filtermZlatest�set�sortedr Zarch�addZ matched_keysZmatched_haystacksr' Z matchcallback�logger�info)r- �argsr. �counter�argZneedlesZpkgsr/ Z used_attrsr7 r) Zprint_section_header�limit�seenr )r" r# r- r �_search9 sb � zSearchCommand._searchc C s` d| |i}t j�|�r$d| |i}| jj�� jtjfi |��}|� � D ]}|� |||� qH|S )Nz %s__substrz%s__glob) r �utilZis_glob_patternr& r8 r9 r: �hawkeyZICASE�runr= )r- rA r! ZneedleZfdict�qr/ r r r r3 � s z SearchCommand._search_countedc C s | j js | jjtjtjd� d S )N)�stdout�stderr)r4 �quiet�cliZredirect_logger�loggingZWARNING�INFO�r- r r r � pre_configure� s zSearchCommand.pre_configurec C sD | j js| j�� | jj}d|_d|_d|_| j jp:| j j | j _d S )NTF) r4 rL rM Zredirect_repo_progress�demandsZavailable_reposZfresh_metadataZsack_activationr Zquery_string_action)r- rR r r r � configure� s zSearchCommand.configurec C s t �td�� | �| jj�S )NzSearching Packages: )r> �debugr rE r4 r rP r r r rH � s zSearchCommand.runN)�__name__� __module__� __qualname__�__doc__�aliasesr r � staticmethodr rE r3 rQ rS rH r r r r r ( s O r )Z __future__r r r r1 Zdnf.clir Zdnf.cli.option_parserr Zdnf.i18nr r r r Zdnf.match_counterZdnf.utilrG rN Z getLoggerr> ZCommandr r r r r �<module> s __pycache__/history.cpython-39.opt-1.pyc 0000644 00000026315 15025176534 0014064 0 ustar 00 a �!c%F � @ s� d dl mZ d dl mZ d dl mZ d dlZd dlZd dlmZmZ d dl m Z d dl m Z m Z d dl Zd dlZd dlZd dlZd dlZd dlZd dlZe�d�ZG d d � d e j�ZdS ) � )�absolute_import)�print_function)�unicode_literalsN)�_�ucd)�commands)�TransactionReplay�serialize_transaction�dnfc s� e Zd ZdZdZed�Zg d�Z� fdd�Ze dd� �Z d d � Z d d � Z d d� Z dd� Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Z� ZS )#�HistoryCommandzUA class containing methods needed by the cli to execute the history command. )�historyZhistz(display, or use, the transaction history)�list�info�redo�replay�rollback�store�undo� userinstalledc s t t| �j|i |�� d| _d S )NF)�superr �__init__�_require_one_transaction_id)�self�args�kw�� __class__� �</usr/lib/python3.9/site-packages/dnf/cli/commands/history.pyr 4 s zHistoryCommand.__init__c C s� | j dddd�tjd d�tjdd � ��d� | j d d d d � | j d dd td�d� | j dd td�d � | j dd td�d � | j dd td�d � | j ddddd� | j ddd d d� d S ) N�transactions_action�?ZCOMMANDz$Available commands: {} (default), {}r z, � )�nargs�metavar�helpz --reverse� store_truez$display history list output reversed)�actionr$ z-oz--outputz<For the store command, file path to store the transaction to)�defaultr$ z--ignore-installedzXFor the replay command, don't check for installed packages matching those in transactionz--ignore-extraszRFor the replay command, don't check for extra packages pulled into the transactionz--skip-unavailablezYFor the replay command, skip packages that are not available or have missing dependencies� transactions�*Z TRANSACTIONz�For commands working with history transactions, Transaction ID (<number>, 'last' or 'last-<number>' for one transaction, <transaction-id>..<transaction-id> for a range)�transaction_filenameZTRANSACTION_FILEzEFor the replay command, path to the stored transaction file to replay)� add_argument�formatr �_CMDS�joinr )�parserr r r � set_argparser9 s6 ��� ���� � �z HistoryCommand.set_argparserc C s, | j js| jd | j _n0| j j| jvrH| j j�d| j j� | jd | j _td��| j j�| _| jj }| j jdk�r| j js�t j� td���t | j j�dkr�t j� td���t j�| j jd �| j _g | j _d|_d|_d|_d| jj_d| jj_t jj�| j| j� n�| j jd k�r6d| _| j j�s�t j� td ���n�| j jd v �r�d|_d|_d|_d| _| j j�s�td �}t� |� t j� |��n,t | j j�dk�r�t� | j� t j� | j��d|_t jj�| j| j� nd|_ d|_ | jj jd k�r(t � | jj jt j!��s(td | jj j �}t� |� t j� |��d S )Nr zUFound more than one transaction ID. '{}' requires one transaction ID or package name.r z No transaction file name given.r! z6More than one argument given as transaction file name.TFr z(No transaction ID or package name given.)r r r z:memory:z+You don't have access to the history DB: %s)"�optsr r- r( �insertr r, � _require_one_transaction_id_msg�cli�demandsr �CliError�len�os�path�abspathr* Zavailable_reposZ resolvingZ root_user�base�confZ clean_requirements_on_removeZinstall_weak_depsr Z _checkGPGKeyr �logger�criticalZfresh_metadataZsack_activationr �access�R_OK)r r5 �msgr r r � configureU s\ � & zHistoryCommand.configurec C s� t |tjj�rv| jjdkr2| jj\}td�| fS | jjdkrv| jjd dkrV| jjn| jjdd� \}td�| fS tjj j � | |�S ) z.Get suggestions for resolving the given error.r zVCannot undo transaction %s, doing so would result in an inconsistent package database.r r �forcer! NzZCannot rollback transaction %s, doing so would result in an inconsistent package database.) � isinstancer � exceptionsZTransactionCheckErrorr1 r r( r r4 r �Command�get_error_output)r �errorZid_r r r rG � s � ��z HistoryCommand.get_error_outputc C s: | � |�}t|�}t| j|dd| jjd�| _| j�� d S )NT��data�ignore_installed� ignore_extras�skip_unavailable)�_history_get_transactionr r r; r1 rM r �run)r �extcmds�oldrJ r r r � _hcmd_redo� s �zHistoryCommand._hcmd_redoc C sD |st j�td���| jj�|�}|s@t j�td��|d ���|S )NzNo transaction ID givenz Transaction ID "{0}" not found.r )r r4 r6 r r; r rQ r, �r rP rQ r r r �_history_get_transactions� s z(HistoryCommand._history_get_transactionsc C s. | � |�}t|�dkr&tj�td���|d S )Nr! z#Found more than one transaction ID!r )rT r7 r r4 r6 r rS r r r rN � s z'HistoryCommand._history_get_transactionc C s | � |�}| �|� d S )N)rN �_revert_transactionrS r r r � _hcmd_undo� s zHistoryCommand._hcmd_undoc C s� | � |�}| jj�� }d }|j|jkr�| jj�tt|jd |jd ���D ]X}|jrht � t d�|j� n|j r�t � t d�|j� |d u r�t jj�|�}qJ|�|� qJ| �|� d S )Nr! z-Transaction history is incomplete, before %u.z,Transaction history is incomplete, after %u.)rN r; r �last�tidrQ r �rangeZaltered_lt_rpmdbr= Zwarningr Zaltered_gt_rpmdbr ZdbZMergedTransactionWrapper�mergerU )r rP rQ rW Z merged_trans�transr r r �_hcmd_rollback� s ( z HistoryCommand._hcmd_rollbackc C s dddddddddd d d � }t |�}d D ]�}|�|g �D ]�}||d |d < |d dkrp|�dd �dkrpd|d< |d d kr�d|v r�t�|d �}|jtjgd�d }| jjj� |j |j |� � d d �}t j�|�|d< |�d�tjkr8d |d< q8q(t| j|dd| jjd�| _| j�� d S )N�Removed�Install� Downgraded� Downgrade�Upgraded�Upgrade� Reinstall� Reinstalled� Obsoleted� Reason Change) r^ r] rb ra r` r_ rd rc re ZObsoleterf )Zrpms�groupsZ environmentsr& �reasonZcleanZ dependency�nevra)Zformsr r! Zrepo_idTrI )r �get�hawkeyZSubjectZget_nevra_possibilitiesZ FORM_NEVRA�outputr ZswdbZ resolveRPMTransactionItemReason�nameZarch�tids�libdnfZ transactionZ TransactionItemReasonToStringZSYSTEM_REPO_NAMEr r; r1 rM r rO ) r r[ Z action_maprJ Z content_typeZtiZsubjri rh r r r rU � sJ � � �z"HistoryCommand._revert_transactionc C s: t | j�� �}| j�|dd�}|dkr6tj�td���dS )z&Execute history userinstalled command.zPackages installed by userri r zNo packages to listN) �tupler; Ziter_userinstalledrl ZlistPkgsr r4 r6 r )r ZpkgsZn_listedr r r �_hcmd_userinstalled s z"HistoryCommand._hcmd_userinstalledc s� � fdd�}t � }t � }� jjD �]�}d|v �rRz|�dd�\}}W n. tyr t�td��|�� t j j �Y n0 td�}z ||�}W n. ty� t�t|��|�� t j j �Y n0 z ||�}W n. ty� t�t|��|�� t j j �Y n0 � j �r||k�rt�� j � t j j �||k�r.|| }}|�||f� |�t||d �� q z|�||�� W q t�y� � jj�|g�}|�r�|�|� n4td��|�} � j �r�t�| � t j j �n t�| � Y q 0 q t|d d �|fS ) z0Convert commandline arguments to transaction idsc sJ | dkrd} n| � d�r$| dd � } t| �}|dkrF|� jj�� j7 }|S )NrW �0zlast-� r )� startswith�intrl r rW rX )�sZtransaction_id�r r r �str2transaction_id s z@HistoryCommand._args2transaction_ids.<locals>.str2transaction_idz..� zWInvalid transaction ID range definition '{}'. Use '<transaction-id>..<transaction-id>'.zNCan't convert '{}' to transaction ID. Use '<number>', 'last', 'last-<number>'.r! z8No transaction which manipulates package '{}' was found.T��reverse)�setr1 r( �split� ValueErrorr= r> r r, r r4 r6 r r3 �add�updaterY rl r �searchr �sorted) r rx rn � merged_tids�tZbegin_transaction_idZend_transaction_idZcant_convert_msgZtransact_ids_from_pkgnamerA r rw r �_args2transaction_ids s^ �� � � z$HistoryCommand._args2transaction_idsc C sP | j j}|dkrDt| j| j j| j j| j j| j jd�| _| j� � �n| � � \}}|dkr||sd| j j s|| j j || j jd� �n�|dkr�|s�| j j s�| j �|| j j |� �n�|dkr�| �|� �n�|dkr�| �|� �nx|dkr�| �|� �nb|d kr�| �� �nN|d k�rL| �|�}t|�}z�| j j d u�r2| j j nd }| jjj�sN| jjj�s�tj�|��r�t d �� |�}| jjj�s�| jj j d � |�d� |�d��s�t t d�� |�� W d S t |d��,}t!j"||ddd� |�#d� W d � n1 �s�0 Y t t d�� |�� W n@ t$�yJ } z&t%j&�'t d�� t(| ����W Y d } ~ n d } ~ 0 0 d S )Nr )�filenamerK rL rM r rz r r r r r r ztransaction.jsonz{} exists, overwrite?z {} [y/N]: z {} [Y/n]: )rA Zdefaultyes_msgz Not overwriting {}, exiting.�wrs T)�indentZ sort_keys� zTransaction saved to {}.z Error storing transaction: {}))r1 r r r; r* rK rL rM r rO r� r( rl ZhistoryListCmdr{ ZhistoryInfoCmdrV rR r\ rq rN r r< ZassumenoZ assumeyesr8 r9 �isfiler r, Z userconfirm�print�open�json�dump�write�OSErrorr r4 r6 �str) r Zvcmdrn r� rX rJ r� rA �f�er r r rO M sR � &� *zHistoryCommand.runc C s | j jdvrd S | j�� d S )N�r r r r )r1 r r Zpost_transactionrw r r r � run_resolved s zHistoryCommand.run_resolvedc C sT | j jdvrd S | j�� }|rPt�tjjt d�� |D ]}t�tjjd| � q6d S )Nr� zEWarning, the following problems occurred while running a transaction:z ) r1 r r Z get_warningsr= �logr �loggingZWARNINGr )r �warningsr� r r r �run_transaction� s �z HistoryCommand.run_transaction)�__name__� __module__� __qualname__�__doc__�aliasesr Zsummaryr- r � staticmethodr0 rB rG rR rT rN rV r\ rU rq r� rO r� r� � __classcell__r r r r r * s( = 0@2r )Z __future__r r r ro rk Zdnf.i18nr r Zdnf.clir Zdnf.transaction_srr r r Zdnf.exceptionsZdnf.transactionZdnf.utilr� r� r8 Z getLoggerr= rF r r r r r �<module> s upgrademinimal.py 0000644 00000003407 15025176534 0010130 0 ustar 00 # # Copyright (C) 2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.i18n import _ from dnf.cli.commands.upgrade import UpgradeCommand class UpgradeMinimalCommand(UpgradeCommand): """A class containing methods needed by the cli to execute the check command. """ aliases = ('upgrade-minimal', 'update-minimal', 'up-min') summary = _("upgrade, but only 'newest' package match which fixes a problem" " that affects your system") def configure(self): UpgradeCommand.configure(self) self.upgrade_minimal = True if not any([self.opts.bugfix, self.opts.enhancement, self.opts.newpackage, self.opts.security, self.opts.advisory, self.opts.bugzilla, self.opts.cves, self.opts.severity]): self.all_security = True swap.py 0000644 00000005101 15025176534 0006075 0 ustar 00 # # Copyright (C) 2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.i18n import _ from dnf.cli import commands import dnf.util import logging logger = logging.getLogger("dnf") class SwapCommand(commands.Command): """A class containing methods needed by the cli to execute the swap command. """ aliases = ('swap',) summary = _('run an interactive {prog} mod for remove and install one spec').format( prog=dnf.util.MAIN_PROG_UPPER) @staticmethod def set_argparser(parser): parser.add_argument('remove_spec', action="store", help=_('The specs that will be removed')) parser.add_argument('install_spec', action="store", help=_( 'The specs that will be installed')) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) commands._checkEnabledRepo(self.base, [self.opts.install_spec]) def _perform(self, cmd_str, spec): cmd_cls = self.cli.cli_commands.get(cmd_str) if cmd_cls is not None: cmd = cmd_cls(self.cli) self.cli.optparser.parse_command_args(cmd, [cmd_str, spec]) cmd.run() def run(self): # The install part must be performed before the remove one because it can # operate on local rpm files. Command line packages cannot be added # to the sack once the goal is created. self._perform('install', self.opts.install_spec) self._perform('remove', self.opts.remove_spec) alias.py 0000644 00000015660 15025176534 0006227 0 ustar 00 # alias.py # Alias CLI command. # # Copyright (C) 2018 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import logging import os.path import dnf.cli import dnf.cli.aliases from dnf.cli import commands import dnf.conf import dnf.exceptions from dnf.i18n import _ logger = logging.getLogger('dnf') class AliasCommand(commands.Command): aliases = ('alias',) summary = _('List or create command aliases') @staticmethod def set_argparser(parser): enable_group = parser.add_mutually_exclusive_group() enable_group.add_argument( '--enable-resolving', default=False, action='store_true', help=_('enable aliases resolving')) enable_group.add_argument( '--disable-resolving', default=False, action='store_true', help=_('disable aliases resolving')) parser.add_argument("subcommand", nargs='?', default='list', choices=['add', 'list', 'delete'], help=_("action to do with aliases")) parser.add_argument("alias", nargs="*", metavar="command[=result]", help=_("alias definition")) def configure(self): demands = self.cli.demands if self.opts.subcommand in ('add', 'delete'): demands.root_user = True self.aliases_base = dnf.cli.aliases.Aliases() self.aliases_base._load_aliases() self.resolving_enabled = self.aliases_base.enabled self._update_config_from_options() def _update_config_from_options(self): enabled = None if self.opts.enable_resolving: enabled = True logger.info(_("Aliases are now enabled")) if self.opts.disable_resolving: enabled = False logger.info(_("Aliases are now disabled")) if enabled is not None: if not os.path.exists(dnf.cli.aliases.ALIASES_CONF_PATH): open(dnf.cli.aliases.ALIASES_CONF_PATH, 'w').close() dnf.conf.BaseConfig.write_raw_configfile( dnf.cli.aliases.ALIASES_CONF_PATH, 'main', None, {'enabled': enabled}) if not self.aliases_base._disabled_by_environ(): self.aliases_base.enabled = enabled def _parse_option_alias(self): new_aliases = {} for alias in self.opts.alias: alias = alias.split('=', 1) cmd = alias[0].strip() if len(cmd.split()) != 1: logger.warning(_("Invalid alias key: %s"), cmd) continue if cmd.startswith('-'): logger.warning(_("Invalid alias key: %s"), cmd) continue if len(alias) == 1: logger.warning(_("Alias argument has no value: %s"), cmd) continue new_aliases[cmd] = alias[1].split() return new_aliases def _load_user_aliases(self): if not os.path.exists(dnf.cli.aliases.ALIASES_USER_PATH): open(dnf.cli.aliases.ALIASES_USER_PATH, 'w').close() try: conf = dnf.cli.aliases.AliasesConfig( dnf.cli.aliases.ALIASES_USER_PATH) except dnf.exceptions.ConfigError as e: logger.warning(_('Config error: %s'), e) return None return conf def _store_user_aliases(self, user_aliases, enabled): fileobj = open(dnf.cli.aliases.ALIASES_USER_PATH, 'w') output = "[main]\n" output += "enabled = {}\n\n".format(enabled) output += "[aliases]\n" for key, value in user_aliases.items(): output += "{} = {}\n".format(key, ' '.join(value)) fileobj.write(output) def add_aliases(self, aliases): conf = self._load_user_aliases() user_aliases = conf.aliases if user_aliases is None: return user_aliases.update(aliases) self._store_user_aliases(user_aliases, conf.enabled) logger.info(_("Aliases added: %s"), ', '.join(aliases.keys())) def remove_aliases(self, cmds): conf = self._load_user_aliases() user_aliases = conf.aliases if user_aliases is None: return valid_cmds = [] for cmd in cmds: try: del user_aliases[cmd] valid_cmds.append(cmd) except KeyError: logger.info(_("Alias not found: %s"), cmd) self._store_user_aliases(user_aliases, conf.enabled) logger.info(_("Aliases deleted: %s"), ', '.join(valid_cmds)) def list_alias(self, cmd): args = [cmd] try: args = self.aliases_base._resolve(args) except dnf.exceptions.Error as e: logger.error( _('%s, alias %s="%s"'), e, cmd, (' ').join(self.aliases_base.aliases[cmd])) else: print(_("Alias %s='%s'") % (cmd, " ".join(args))) def run(self): if not self.aliases_base.enabled: logger.warning(_("Aliases resolving is disabled.")) if self.opts.subcommand == 'add': # Add new alias aliases = self._parse_option_alias() if not aliases: raise dnf.exceptions.Error(_("No aliases specified.")) self.add_aliases(aliases) return if self.opts.subcommand == 'delete': # Remove alias cmds = self.opts.alias if cmds == []: raise dnf.exceptions.Error(_("No alias specified.")) self.remove_aliases(cmds) return if not self.opts.alias: # List all aliases if not self.aliases_base.aliases: logger.info(_("No aliases defined.")) return for cmd in self.aliases_base.aliases: self.list_alias(cmd) else: # List alias by key for cmd in self.opts.alias: if cmd not in self.aliases_base.aliases: logger.info(_("No match for alias: %s") % cmd) continue self.list_alias(cmd) deplist.py 0000644 00000002746 15025176534 0006603 0 ustar 00 # # Copyright (C) 2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import print_function from __future__ import absolute_import from __future__ import unicode_literals from dnf.i18n import _ from dnf.cli.commands.repoquery import RepoQueryCommand class DeplistCommand(RepoQueryCommand): """ The command is alias for 'dnf repoquery --deplist' """ aliases = ('deplist',) summary = _("[deprecated, use repoquery --deplist] List package's dependencies and what packages provide them") def configure(self): RepoQueryCommand.configure(self) self.opts.deplist = True reinstall.py 0000644 00000010135 15025176534 0007123 0 ustar 00 # reinstall.py # Reinstall CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _ import dnf.exceptions import logging logger = logging.getLogger('dnf') class ReinstallCommand(commands.Command): """A class containing methods needed by the cli to execute the reinstall command. """ aliases = ('reinstall', 'rei') summary = _('reinstall a package') @staticmethod def set_argparser(parser): parser.add_argument('packages', nargs='+', help=_('Package to reinstall'), action=OptionParser.ParseSpecGroupFileCallback, metavar=_('PACKAGE')) def configure(self): """Verify that conditions are met so that this command can run. These include that the program is being run by the root user, that there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. """ demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) if not self.opts.filenames: commands._checkEnabledRepo(self.base) def run(self): # Reinstall files. done = False for pkg in self.base.add_remote_rpms(self.opts.filenames, strict=False, progress=self.base.output.progress): try: self.base.package_reinstall(pkg) except dnf.exceptions.MarkingError: logger.info(_('No match for argument: %s'), self.base.output.term.bold(pkg.location)) else: done = True # Reinstall packages. for pkg_spec in self.opts.pkg_specs + ['@' + x for x in self.opts.grp_specs]: try: self.base.reinstall(pkg_spec) except dnf.exceptions.PackagesNotInstalledError as err: for pkg in err.packages: logger.info(_('Package %s available, but not installed.'), self.output.term.bold(pkg.name)) break logger.info(_('No match for argument: %s'), self.base.output.term.bold(pkg_spec)) except dnf.exceptions.PackagesNotAvailableError as err: for pkg in err.packages: xmsg = '' pkgrepo = self.base.history.repo(pkg) if pkgrepo: xmsg = _(' (from %s)') % pkgrepo msg = _('Installed package %s%s not available.') logger.info(msg, self.base.output.term.bold(pkg), xmsg) except dnf.exceptions.MarkingError: assert False, 'Only the above marking errors are expected.' else: done = True if not done: raise dnf.exceptions.Error(_('No packages marked for reinstall.')) group.py 0000644 00000035161 15025176534 0006270 0 ustar 00 # group.py # Group CLI command. # # Copyright (C) 2012-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.comps import CompsQuery from dnf.cli import commands from dnf.i18n import _, ucd import libdnf.transaction import dnf.cli import dnf.exceptions import dnf.util import logging logger = logging.getLogger("dnf") class GroupCommand(commands.Command): """ Single sub-command interface for most groups interaction. """ direct_commands = {'grouplist' : 'list', 'groupinstall' : 'install', 'groupupdate' : 'install', 'groupremove' : 'remove', 'grouperase' : 'remove', 'groupinfo' : 'info'} aliases = ('group', 'groups', 'grp') + tuple(direct_commands.keys()) summary = _('display, or use, the groups information') _CMD_ALIASES = {'update' : 'upgrade', 'erase' : 'remove'} _MARK_CMDS = ('install', 'remove') _GROUP_SUBCOMMANDS = ('summary', 'list', 'info', 'remove', 'install', 'upgrade', 'mark') def _canonical(self): # were we called with direct command? direct = self.direct_commands.get(self.opts.command) if direct: # canonize subcmd and args if self.opts.subcmd is not None: self.opts.args.insert(0, self.opts.subcmd) self.opts.subcmd = direct if self.opts.subcmd is None: self.opts.subcmd = 'summary' self.opts.subcmd = self._CMD_ALIASES.get(self.opts.subcmd, self.opts.subcmd) def __init__(self, cli): super(GroupCommand, self).__init__(cli) self._remark = False def _assert_comps(self): msg = _('No group data available for configured repositories.') if not len(self.base.comps): raise dnf.exceptions.CompsError(msg) def _environment_lists(self, patterns): def available_pred(env): env_found = self.base.history.env.get(env.id) return not(env_found) self._assert_comps() if patterns is None: envs = self.base.comps.environments else: envs = self.base.comps.environments_by_pattern(",".join(patterns)) return dnf.util.mapall(list, dnf.util.partition(available_pred, envs)) def _group_lists(self, uservisible, patterns): def installed_pred(group): group_found = self.base.history.group.get(group.id) if group_found: return True return False installed = [] available = [] self._assert_comps() if patterns is None: grps = self.base.comps.groups else: grps = self.base.comps.groups_by_pattern(",".join(patterns)) for grp in grps: tgt_list = available if installed_pred(grp): tgt_list = installed if not uservisible or grp.uservisible: tgt_list.append(grp) return installed, available def _info(self, userlist): for strng in userlist: group_matched = False for env in self.base.comps.environments_by_pattern(strng): self.output.display_groups_in_environment(env) group_matched = True for group in self.base.comps.groups_by_pattern(strng): self.output.display_pkgs_in_groups(group) group_matched = True if not group_matched: logger.error(_('Warning: Group %s does not exist.'), strng) return 0, [] def _list(self, userlist): uservisible = 1 showinstalled = 0 showavailable = 0 print_ids = self.base.conf.verbose or self.opts.ids while userlist: if userlist[0] == 'hidden': uservisible = 0 userlist.pop(0) elif userlist[0] == 'installed': showinstalled = 1 userlist.pop(0) elif userlist[0] == 'available': showavailable = 1 userlist.pop(0) elif userlist[0] == 'ids': print_ids = True userlist.pop(0) else: break if self.opts.hidden: uservisible = 0 if self.opts.installed: showinstalled = 1 if self.opts.available: showavailable = 1 if not userlist: userlist = None # Match everything... errs = False if userlist is not None: for group in userlist: comps = self.base.comps in_group = len(comps.groups_by_pattern(group)) > 0 in_environment = len(comps.environments_by_pattern(group)) > 0 if not in_group and not in_environment: logger.error(_('Warning: No groups match:') + '\n %s', group) errs = True if errs: return 0, [] env_inst, env_avail = self._environment_lists(userlist) installed, available = self._group_lists(uservisible, userlist) def _out_grp(sect, group): if not done: print(sect) msg = ' %s' % dnf.util._name_unset_wrapper(group.ui_name) if print_ids: msg += ' (%s)' % group.id if group.lang_only: msg += ' [%s]' % group.lang_only print('{}'.format(msg)) def _out_env(sect, envs): if envs: print(sect) for e in envs: msg = ' %s' % dnf.util._name_unset_wrapper(e.ui_name) if print_ids: msg += ' (%s)' % e.id print(msg) if not showinstalled: _out_env(_('Available Environment Groups:'), env_avail) if not showavailable: _out_env(_('Installed Environment Groups:'), env_inst) if not showavailable: done = False for group in installed: if group.lang_only: continue _out_grp(_('Installed Groups:'), group) done = True done = False for group in installed: if not group.lang_only: continue _out_grp(_('Installed Language Groups:'), group) done = True if showinstalled: return 0, [] done = False for group in available: if group.lang_only: continue _out_grp(_('Available Groups:'), group) done = True done = False for group in available: if not group.lang_only: continue _out_grp(_('Available Language Groups:'), group) done = True return 0, [] def _mark_install(self, patterns): q = CompsQuery(self.base.comps, self.base.history, CompsQuery.GROUPS | CompsQuery.ENVIRONMENTS, CompsQuery.AVAILABLE | CompsQuery.INSTALLED) solver = self.base._build_comps_solver() res = q.get(*patterns) if self.opts.with_optional: types = tuple(self.base.conf.group_package_types + ['optional']) else: types = tuple(self.base.conf.group_package_types) pkg_types = libdnf.transaction.listToCompsPackageType(types) for env_id in res.environments: solver._environment_install(env_id, pkg_types) for group_id in res.groups: solver._group_install(group_id, pkg_types) def _mark_remove(self, patterns): q = CompsQuery(self.base.comps, self.base.history, CompsQuery.GROUPS | CompsQuery.ENVIRONMENTS, CompsQuery.INSTALLED) solver = self.base._build_comps_solver() res = q.get(*patterns) for env_id in res.environments: assert dnf.util.is_string_type(env_id) solver._environment_remove(env_id) for grp_id in res.groups: assert dnf.util.is_string_type(grp_id) solver._group_remove(grp_id) def _mark_subcmd(self, extcmds): if extcmds[0] in self._MARK_CMDS: return extcmds[0], extcmds[1:] return 'install', extcmds def _summary(self, userlist): uservisible = 1 if len(userlist) > 0: if userlist[0] == 'hidden': uservisible = 0 userlist.pop(0) if self.opts.hidden: uservisible = 0 if not userlist: userlist = None # Match everything... installed, available = self._group_lists(uservisible, userlist) def _out_grp(sect, num): if not num: return logger.info('%s %u', sect, num) done = 0 for group in installed: if group.lang_only: continue done += 1 _out_grp(_('Installed Groups:'), done) done = 0 for group in installed: if not group.lang_only: continue done += 1 _out_grp(_('Installed Language Groups:'), done) done = False for group in available: if group.lang_only: continue done += 1 _out_grp(_('Available Groups:'), done) done = False for group in available: if not group.lang_only: continue done += 1 _out_grp(_('Available Language Groups:'), done) return 0, [] @staticmethod def set_argparser(parser): parser.add_argument('--with-optional', action='store_true', help=_("include optional packages from group")) grpparser = parser.add_mutually_exclusive_group() grpparser.add_argument('--hidden', action='store_true', help=_("show also hidden groups")) grpparser.add_argument('--installed', action='store_true', help=_("show only installed groups")) grpparser.add_argument('--available', action='store_true', help=_("show only available groups")) grpparser.add_argument('--ids', action='store_true', help=_("show also ID of groups")) parser.add_argument('subcmd', nargs='?', metavar='COMMAND', help=_('available subcommands: {} (default), {}').format( GroupCommand._GROUP_SUBCOMMANDS[0], ', '.join(GroupCommand._GROUP_SUBCOMMANDS[1:]))) parser.add_argument('args', nargs='*', metavar='COMMAND_ARG', help=_('argument for group subcommand')) def configure(self): self._canonical() cmd = self.opts.subcmd args = self.opts.args if cmd not in self._GROUP_SUBCOMMANDS: logger.critical(_('Invalid groups sub-command, use: %s.'), ", ".join(self._GROUP_SUBCOMMANDS)) raise dnf.cli.CliError if cmd in ('install', 'remove', 'mark', 'info') and not args: self.cli.optparser.print_help(self) raise dnf.cli.CliError demands = self.cli.demands demands.sack_activation = True if cmd in ('install', 'mark', 'remove', 'upgrade'): demands.root_user = True demands.resolving = True if cmd == 'remove': demands.allow_erasing = True demands.available_repos = False else: demands.available_repos = True if cmd not in ('remove'): commands._checkEnabledRepo(self.base) if cmd in ('install', 'upgrade'): commands._checkGPGKey(self.base, self.cli) def run(self): cmd = self.opts.subcmd extcmds = self.opts.args if cmd == 'summary': return self._summary(extcmds) if cmd == 'list': return self._list(extcmds) if cmd == 'info': return self._info(extcmds) if cmd == 'mark': (subcmd, extcmds) = self._mark_subcmd(extcmds) if subcmd == 'remove': return self._mark_remove(extcmds) else: assert subcmd == 'install' return self._mark_install(extcmds) if cmd == 'install': if self.opts.with_optional: types = tuple(self.base.conf.group_package_types + ['optional']) else: types = tuple(self.base.conf.group_package_types) self._remark = True try: return self.base.env_group_install(extcmds, types, self.base.conf.strict) except dnf.exceptions.MarkingError as e: msg = _('No package %s available.') logger.info(msg, self.base.output.term.bold(e)) raise dnf.exceptions.PackagesNotAvailableError( _("Unable to find a mandatory group package.")) if cmd == 'upgrade': return self.base.env_group_upgrade(extcmds) if cmd == 'remove': for arg in extcmds: try: self.base.env_group_remove([arg]) except dnf.exceptions.Error: pass def run_transaction(self): if not self._remark: return goal = self.base._goal history = self.base.history names = goal.group_members for pkg in self.base.sack.query().installed().filterm(name=names): reason = history.rpm.get_reason(pkg) history.set_reason(pkg, goal.group_reason(pkg, reason)) distrosync.py 0000644 00000003637 15025176534 0007340 0 ustar 00 # distrosync.py # distro-sync CLI command. # # Copyright (C) 2012-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from dnf.cli import commands from dnf.i18n import _ class DistroSyncCommand(commands.Command): """A class containing methods needed by the cli to execute the distro-synch command. """ aliases = ('distro-sync', 'distrosync', 'distribution-synchronization', 'dsync') summary = _('synchronize installed packages to the latest available versions') @staticmethod def set_argparser(parser): parser.add_argument('package', nargs='*', help=_('Package to synchronize')) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) commands._checkEnabledRepo(self.base, self.opts.package) def run(self): return self.base.distro_sync_userlist(self.opts.package) search.py 0000644 00000014242 15025176534 0006376 0 ustar 00 # search.py # Search CLI command. # # Copyright (C) 2012-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import collections from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import ucd, _, C_ import dnf.i18n import dnf.match_counter import dnf.util import hawkey import logging logger = logging.getLogger('dnf') class SearchCommand(commands.Command): """A class containing methods needed by the cli to execute the search command. """ aliases = ('search', 'se') summary = _('search package details for the given string') @staticmethod def set_argparser(parser): parser.add_argument('--all', action='store_true', help=_("search also package description and URL")) parser.add_argument('query_string', nargs='+', metavar=_('KEYWORD'), choices=['all'], default=None, action=OptionParser.PkgNarrowCallback, help=_("Keyword to search for")) def _search(self, args): """Search for simple text tags in a package object.""" TRANS_TBL = collections.OrderedDict(( ('name', C_('long', 'Name')), ('summary', C_('long', 'Summary')), ('description', C_('long', 'Description')), ('url', _('URL')), )) def _translate_attr(attr): try: return TRANS_TBL[attr] except: return attr def _print_section_header(exact_match, attrs, keys): trans_attrs = map(_translate_attr, attrs) # TRANSLATORS: separator used between package attributes (eg. Name & Summary & URL) trans_attrs_str = _(' & ').join(trans_attrs) if exact_match: # TRANSLATORS: %s - translated package attributes, # %%s - found keys (in listed attributes) section_text = _('%s Exactly Matched: %%s') % trans_attrs_str else: # TRANSLATORS: %s - translated package attributes, # %%s - found keys (in listed attributes) section_text = _('%s Matched: %%s') % trans_attrs_str formatted = self.base.output.fmtSection(section_text % ", ".join(keys)) print(ucd(formatted)) counter = dnf.match_counter.MatchCounter() for arg in args: self._search_counted(counter, 'name', arg) self._search_counted(counter, 'summary', arg) if self.opts.all: for arg in args: self._search_counted(counter, 'description', arg) self._search_counted(counter, 'url', arg) else: needles = len(args) pkgs = list(counter.keys()) for pkg in pkgs: if len(counter.matched_needles(pkg)) != needles: del counter[pkg] used_attrs = None matched_needles = None exact_match = False print_section_header = False limit = None if not self.base.conf.showdupesfromrepos: limit = self.base.sack.query().filterm(pkg=counter.keys()).latest() seen = set() for pkg in counter.sorted(reverse=True, limit_to=limit): if not self.base.conf.showdupesfromrepos: if pkg.name + pkg.arch in seen: continue seen.add(pkg.name + pkg.arch) if used_attrs != counter.matched_keys(pkg): used_attrs = counter.matched_keys(pkg) print_section_header = True if matched_needles != counter.matched_needles(pkg): matched_needles = counter.matched_needles(pkg) print_section_header = True if exact_match != (counter.matched_haystacks(pkg) == matched_needles): exact_match = counter.matched_haystacks(pkg) == matched_needles print_section_header = True if print_section_header: _print_section_header(exact_match, used_attrs, matched_needles) print_section_header = False self.base.output.matchcallback(pkg, counter.matched_haystacks(pkg), args) if len(counter) == 0: logger.info(_('No matches found.')) def _search_counted(self, counter, attr, needle): fdict = {'%s__substr' % attr : needle} if dnf.util.is_glob_pattern(needle): fdict = {'%s__glob' % attr : needle} q = self.base.sack.query().filterm(hawkey.ICASE, **fdict) for pkg in q.run(): counter.add(pkg, attr, needle) return counter def pre_configure(self): if not self.opts.quiet: self.cli.redirect_logger(stdout=logging.WARNING, stderr=logging.INFO) def configure(self): if not self.opts.quiet: self.cli.redirect_repo_progress() demands = self.cli.demands demands.available_repos = True demands.fresh_metadata = False demands.sack_activation = True self.opts.all = self.opts.all or self.opts.query_string_action def run(self): logger.debug(_('Searching Packages: ')) return self._search(self.opts.query_string) module.py 0000644 00000040731 15025176534 0006420 0 ustar 00 # supplies the 'module' command. # # Copyright (C) 2014-2017 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import print_function from dnf.cli import commands, CliError from dnf.i18n import _ from dnf.module.exceptions import NoModuleException from dnf.util import logger import dnf.util import sys import os import hawkey import libdnf import dnf.module.module_base import dnf.exceptions class ModuleCommand(commands.Command): class SubCommand(commands.Command): def __init__(self, cli): super(ModuleCommand.SubCommand, self).__init__(cli) self.module_base = dnf.module.module_base.ModuleBase(self.base) def _get_modules_from_name_stream_specs(self): modules_from_specs = set() for module_spec in self.opts.module_spec: __, nsvcap = self.module_base._get_modules(module_spec) # When there is no match, the problem was already reported by module_base.remove() if nsvcap is None: continue name = nsvcap.name if nsvcap.name else "" stream = nsvcap.stream if nsvcap.stream else "" if (nsvcap.version and nsvcap.version != -1) or nsvcap.context: logger.info(_("Only module name, stream, architecture or profile is used. " "Ignoring unneeded information in argument: '{}'").format( module_spec)) arch = nsvcap.arch if nsvcap.arch else "" modules = self.base._moduleContainer.query(name, stream, "", "", arch) modules_from_specs.update(modules) return modules_from_specs def _get_module_artifact_names(self, use_modules, skip_modules): artifacts = set() pkg_names = set() for module in use_modules: if module not in skip_modules: if self.base._moduleContainer.isModuleActive(module): artifacts.update(module.getArtifacts()) for artifact in artifacts: subj = hawkey.Subject(artifact) for nevra_obj in subj.get_nevra_possibilities( forms=[hawkey.FORM_NEVRA]): if nevra_obj.name: pkg_names.add(nevra_obj.name) return pkg_names, artifacts class ListSubCommand(SubCommand): aliases = ('list',) summary = _('list all module streams, profiles and states') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True def run_on_module(self): mods = self.module_base if self.opts.enabled: output = mods._get_brief_description( self.opts.module_spec, libdnf.module.ModulePackageContainer.ModuleState_ENABLED) elif self.opts.disabled: output = mods._get_brief_description( self.opts.module_spec, libdnf.module.ModulePackageContainer.ModuleState_DISABLED) elif self.opts.installed: output = mods._get_brief_description( self.opts.module_spec, libdnf.module.ModulePackageContainer.ModuleState_INSTALLED) else: output = mods._get_brief_description( self.opts.module_spec, libdnf.module.ModulePackageContainer.ModuleState_UNKNOWN) if output: print(output) return if self.opts.module_spec: msg = _('No matching Modules to list') raise dnf.exceptions.Error(msg) class InfoSubCommand(SubCommand): aliases = ('info',) summary = _('print detailed information about a module') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True def run_on_module(self): if self.opts.verbose: output = self.module_base._get_full_info(self.opts.module_spec) elif self.opts.profile: output = self.module_base._get_info_profiles(self.opts.module_spec) else: output = self.module_base._get_info(self.opts.module_spec) if output: print(output) else: raise dnf.exceptions.Error(_('No matching Modules to list')) class EnableSubCommand(SubCommand): aliases = ('enable',) summary = _('enable a module stream') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_module(self): try: self.module_base.enable(self.opts.module_spec) except dnf.exceptions.MarkingErrors as e: if self.base.conf.strict: if e.no_match_group_specs or e.error_group_specs: raise e if e.module_depsolv_errors and e.module_depsolv_errors[1] != \ libdnf.module.ModulePackageContainer.ModuleErrorType_ERROR_IN_DEFAULTS: raise e logger.error(str(e)) class DisableSubCommand(SubCommand): aliases = ('disable',) summary = _('disable a module with all its streams') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_module(self): try: self.module_base.disable(self.opts.module_spec) except dnf.exceptions.MarkingErrors as e: if self.base.conf.strict: if e.no_match_group_specs or e.error_group_specs: raise e if e.module_depsolv_errors and e.module_depsolv_errors[1] != \ libdnf.module.ModulePackageContainer.ModuleErrorType_ERROR_IN_DEFAULTS: raise e logger.error(str(e)) class ResetSubCommand(SubCommand): aliases = ('reset',) summary = _('reset a module') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_module(self): try: self.module_base.reset(self.opts.module_spec) except dnf.exceptions.MarkingErrors as e: if self.base.conf.strict: if e.no_match_group_specs: raise e logger.error(str(e)) class InstallSubCommand(SubCommand): aliases = ('install',) summary = _('install a module profile including its packages') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_module(self): try: self.module_base.install(self.opts.module_spec, self.base.conf.strict) except dnf.exceptions.MarkingErrors as e: if self.base.conf.strict: if e.no_match_group_specs or e.error_group_specs: raise e logger.error(str(e)) class UpdateSubCommand(SubCommand): aliases = ('update',) summary = _('update packages associated with an active stream') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True def run_on_module(self): module_specs = self.module_base.upgrade(self.opts.module_spec) if module_specs: raise NoModuleException(", ".join(module_specs)) class RemoveSubCommand(SubCommand): aliases = ('remove', 'erase',) summary = _('remove installed module profiles and their packages') def configure(self): demands = self.cli.demands demands.allow_erasing = True demands.available_repos = True demands.fresh_metadata = False demands.resolving = True demands.root_user = True demands.sack_activation = True def run_on_module(self): skipped_groups = self.module_base.remove(self.opts.module_spec) if self.opts.all: modules_from_specs = self._get_modules_from_name_stream_specs() remove_names_from_spec, __ = self._get_module_artifact_names( modules_from_specs, set()) keep_names, __ = self._get_module_artifact_names( self.base._moduleContainer.getModulePackages(), modules_from_specs) remove_query = self.base.sack.query().installed().filterm( name=remove_names_from_spec) keep_query = self.base.sack.query().installed().filterm(name=keep_names) for pkg in remove_query: if pkg in keep_query: msg = _("Package {} belongs to multiple modules, skipping").format(pkg) logger.info(msg) else: self.base.goal.erase( pkg, clean_deps=self.base.conf.clean_requirements_on_remove) if not skipped_groups: return logger.error(dnf.exceptions.MarkingErrors(no_match_group_specs=skipped_groups)) class SwitchToSubCommand(SubCommand): aliases = ('switch-to',) summary = _('switch a module to a stream and distrosync rpm packages') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True demands.resolving = True demands.root_user = True self.base.conf.module_stream_switch = True def run_on_module(self): try: self.module_base.switch_to(self.opts.module_spec, strict=self.base.conf.strict) except dnf.exceptions.MarkingErrors as e: if self.base.conf.strict: if e.no_match_group_specs or e.error_group_specs: raise e logger.error(str(e)) class ProvidesSubCommand(SubCommand): aliases = ("provides", ) summary = _('list modular packages') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True def run_on_module(self): output = self.module_base._what_provides(self.opts.module_spec) if output: print(output) class RepoquerySubCommand(SubCommand): aliases = ("repoquery", ) summary = _('list packages belonging to a module') def configure(self): demands = self.cli.demands demands.available_repos = True demands.sack_activation = True def run_on_module(self): modules_from_specs = set() for module_spec in self.opts.module_spec: modules, __ = self.module_base._get_modules(module_spec) modules_from_specs.update(modules) names_from_spec, spec_artifacts = self._get_module_artifact_names( modules_from_specs, set()) package_strings = set() if self.opts.available or not self.opts.installed: query = self.base.sack.query().available().filterm(nevra_strict=spec_artifacts) for pkg in query: package_strings.add(str(pkg)) if self.opts.installed: query = self.base.sack.query().installed().filterm(name=names_from_spec) for pkg in query: package_strings.add(str(pkg)) output = "\n".join(sorted(package_strings)) print(output) SUBCMDS = {ListSubCommand, InfoSubCommand, EnableSubCommand, DisableSubCommand, ResetSubCommand, InstallSubCommand, UpdateSubCommand, RemoveSubCommand, SwitchToSubCommand, ProvidesSubCommand, RepoquerySubCommand} SUBCMDS_NOT_REQUIRED_ARG = {ListSubCommand} aliases = ("module",) summary = _("Interact with Modules.") def __init__(self, cli): super(ModuleCommand, self).__init__(cli) subcmd_objs = (subcmd(cli) for subcmd in self.SUBCMDS) self.subcmd = None self._subcmd_name2obj = { alias: subcmd for subcmd in subcmd_objs for alias in subcmd.aliases} def set_argparser(self, parser): narrows = parser.add_mutually_exclusive_group() narrows.add_argument('--enabled', dest='enabled', action='store_true', help=_("show only enabled modules")) narrows.add_argument('--disabled', dest='disabled', action='store_true', help=_("show only disabled modules")) narrows.add_argument('--installed', dest='installed', action='store_true', help=_("show only installed modules or packages")) narrows.add_argument('--profile', dest='profile', action='store_true', help=_("show profile content")) parser.add_argument('--available', dest='available', action='store_true', help=_("show only available packages")) narrows.add_argument('--all', dest='all', action='store_true', help=_("remove all modular packages")) subcommand_choices = [] subcommand_help = [] for subcmd in sorted(self.SUBCMDS, key=lambda x: x.aliases[0]): subcommand_choices.append(subcmd.aliases[0]) subcommand_help.append('{}: {}'.format(subcmd.aliases[0], subcmd.summary or '')) parser.add_argument('subcmd', nargs=1, choices=subcommand_choices, metavar='<modular command>', help='\n'.join(subcommand_help)) parser.add_argument('module_spec', metavar='module-spec', nargs='*', help=_("Module specification")) def configure(self): try: self.subcmd = self._subcmd_name2obj[self.opts.subcmd[0]] except (CliError, KeyError): self.cli.optparser.print_usage() raise CliError self.subcmd.opts = self.opts self.subcmd.configure() def run(self): self.check_required_argument() self.subcmd.run_on_module() def check_required_argument(self): not_required_argument = [alias for subcmd in self.SUBCMDS_NOT_REQUIRED_ARG for alias in subcmd.aliases] if self.opts.subcmd[0] not in not_required_argument: if not self.opts.module_spec: raise CliError( _("{} {} {}: too few arguments").format(dnf.util.MAIN_PROG, self.opts.command, self.opts.subcmd[0])) remove.py 0000644 00000014716 15025176534 0006434 0 ustar 00 # remove_command.py # Remove CLI command. # # Copyright (C) 2012-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.i18n import _ from dnf.cli.option_parser import OptionParser import dnf.base import argparse import hawkey import dnf.exceptions import logging logger = logging.getLogger("dnf") class RemoveCommand(commands.Command): """Remove command.""" nevra_forms = {'remove-n': hawkey.FORM_NAME, 'remove-na': hawkey.FORM_NA, 'remove-nevra': hawkey.FORM_NEVRA, 'erase-n': hawkey.FORM_NAME, 'erase-na': hawkey.FORM_NA, 'erase-nevra': hawkey.FORM_NEVRA} aliases = ('remove', 'erase', 'rm') + tuple(nevra_forms.keys()) summary = _('remove a package or packages from your system') @staticmethod def set_argparser(parser): mgroup = parser.add_mutually_exclusive_group() mgroup.add_argument('--duplicates', action='store_true', dest='duplicated', help=_('remove duplicated packages')) mgroup.add_argument('--duplicated', action='store_true', help=argparse.SUPPRESS) mgroup.add_argument('--oldinstallonly', action='store_true', help=_( 'remove installonly packages over the limit')) parser.add_argument('packages', nargs='*', help=_('Package to remove'), action=OptionParser.ParseSpecGroupFileCallback, metavar=_('PACKAGE')) def configure(self): demands = self.cli.demands # disable all available repos to delete whole dependency tree # instead of replacing removable package with available packages demands.resolving = True demands.root_user = True demands.sack_activation = True if self.opts.duplicated: demands.available_repos = True elif dnf.base.WITH_MODULES and self.opts.grp_specs: demands.available_repos = True demands.fresh_metadata = False demands.allow_erasing = True else: demands.allow_erasing = True demands.available_repos = False def run(self): forms = [] if self.opts.command in self.nevra_forms: forms = [self.nevra_forms[self.opts.command]] # local pkgs not supported in erase command self.opts.pkg_specs += self.opts.filenames done = False if self.opts.duplicated: q = self.base.sack.query() instonly = self.base._get_installonly_query(q.installed()) dups = q.duplicated().difference(instonly) if not dups: logger.info(_('No duplicated packages found for removal.')) return for (name, arch), pkgs_list in dups._na_dict().items(): if len(pkgs_list) < 2: continue pkgs_list.sort(reverse=True) try: self.base.reinstall(str(pkgs_list[0])) except dnf.exceptions.PackagesNotAvailableError: xmsg = '' msg = _('Installed package %s%s not available.') logger.warning(msg, self.base.output.term.bold(str(pkgs_list[0])), xmsg) for pkg in pkgs_list[1:]: self.base.package_remove(pkg) return if self.opts.oldinstallonly: q = self.base.sack.query() instonly = self.base._get_installonly_query(q.installed()).latest(-1) # also remove running kernel from the set kernel = self.base.sack.get_running_kernel() if kernel is not None: running_installonly = instonly.filter( epoch=kernel.epoch, version=kernel.version, release=kernel.release) if running_installonly: instonly = instonly.difference(running_installonly) if instonly: for pkg in instonly: self.base.package_remove(pkg) else: logger.info(_('No old installonly packages found for removal.')) return # Remove groups. if self.opts.grp_specs and forms: for grp_spec in self.opts.grp_specs: msg = _('Not a valid form: %s') logger.warning(msg, self.base.output.term.bold(grp_spec)) elif self.opts.grp_specs: if dnf.base.WITH_MODULES: module_base = dnf.module.module_base.ModuleBase(self.base) skipped_grps = module_base.remove(self.opts.grp_specs) if len(self.opts.grp_specs) != len(skipped_grps): done = True else: skipped_grps = self.opts.grp_specs if skipped_grps: for group in skipped_grps: try: if self.base.env_group_remove([group]): done = True except dnf.exceptions.Error: pass for pkg_spec in self.opts.pkg_specs: try: self.base.remove(pkg_spec, forms=forms) except dnf.exceptions.MarkingError as e: msg = '{}: {}'.format(e.value, self.base.output.term.bold(pkg_spec)) logger.info(msg) else: done = True if not done: logger.warning(_('No packages marked for removal.')) upgrade.py 0000644 00000011176 15025176534 0006563 0 ustar 00 # upgrade.py # Upgrade CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals import logging import dnf.exceptions import dnf.base from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _ logger = logging.getLogger('dnf') class UpgradeCommand(commands.Command): """A class containing methods needed by the cli to execute the update command. """ aliases = ('upgrade', 'update', 'upgrade-to', 'update-to', 'localupdate', 'up') summary = _('upgrade a package or packages on your system') @staticmethod def set_argparser(parser): parser.add_argument('packages', nargs='*', help=_('Package to upgrade'), action=OptionParser.ParseSpecGroupFileCallback, metavar=_('PACKAGE')) def configure(self): """Verify that conditions are met so that this command can run. These include that there are enabled repositories with gpg keys, and that this command is being run by the root user. """ demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) if not self.opts.filenames: commands._checkEnabledRepo(self.base) self.upgrade_minimal = None self.all_security = None self.skipped_grp_specs = None def run(self): cmp_type = "eq" if self.upgrade_minimal else "gte" self.cli._populate_update_security_filter(self.opts, cmp_type=cmp_type, all=self.all_security) if self.opts.filenames or self.opts.pkg_specs or self.opts.grp_specs: result = False result |= self._update_modules() result |= self._update_files() result |= self._update_packages() result |= self._update_groups() if result: return else: self.base.upgrade_all() return raise dnf.exceptions.Error(_('No packages marked for upgrade.')) def _update_modules(self): group_specs_num = len(self.opts.grp_specs) if dnf.base.WITH_MODULES: module_base = dnf.module.module_base.ModuleBase(self.base) self.skipped_grp_specs = module_base.upgrade(self.opts.grp_specs) else: self.skipped_grp_specs = self.opts.grp_specs return len(self.skipped_grp_specs) != group_specs_num def _update_files(self): success = False if self.opts.filenames: for pkg in self.base.add_remote_rpms(self.opts.filenames, strict=False, progress=self.base.output.progress): try: self.base.package_upgrade(pkg) success = True except dnf.exceptions.MarkingError as e: logger.info(_('No match for argument: %s'), self.base.output.term.bold(pkg.location)) return success def _update_packages(self): success = False for pkg_spec in self.opts.pkg_specs: try: self.base.upgrade(pkg_spec) success = True except dnf.exceptions.MarkingError as e: logger.info(_('No match for argument: %s'), self.base.output.term.bold(pkg_spec)) return success def _update_groups(self): if self.skipped_grp_specs: self.base.env_group_upgrade(self.skipped_grp_specs) return True return False check.py 0000644 00000016077 15025176534 0006216 0 ustar 00 # # Copyright (C) 2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.i18n import _ from dnf.cli import commands import argparse import dnf.exceptions class CheckCommand(commands.Command): """A class containing methods needed by the cli to execute the check command. """ aliases = ('check',) summary = _('check for problems in the packagedb') @staticmethod def set_argparser(parser): parser.add_argument('--all', dest='check_types', action='append_const', const='all', help=_('show all problems; default')) parser.add_argument('--dependencies', dest='check_types', action='append_const', const='dependencies', help=_('show dependency problems')) parser.add_argument('--duplicates', dest='check_types', action='append_const', const='duplicates', help=_('show duplicate problems')) parser.add_argument('--obsoleted', dest='check_types', action='append_const', const='obsoleted', help=_('show obsoleted packages')) parser.add_argument('--provides', dest='check_types', action='append_const', const='provides', help=_('show problems with provides')) # Add compatibility with yum but invisible in help # In choices [] allows to return empty list if no argument otherwise it fails parser.add_argument('check_yum_types', nargs='*', choices=[ 'all', 'dependencies', 'duplicates', 'obsoleted', 'provides', []], help=argparse.SUPPRESS) def configure(self): self.cli.demands.sack_activation = True if self.opts.check_yum_types: if self.opts.check_types: self.opts.check_types = self.opts.check_types + \ self.opts.check_yum_types else: self.opts.check_types = self.opts.check_yum_types if not self.opts.check_types: self.opts.check_types = {'all'} else: self.opts.check_types = set(self.opts.check_types) self.base.conf.disable_excludes += ["all"] def run(self): output_set = set() q = self.base.sack.query().installed() if self.opts.check_types.intersection({'all', 'dependencies'}): sack = None for pkg in q: for require in set(pkg.regular_requires) | set(set(pkg.requires_pre) - set(pkg.prereq_ignoreinst)): if str(require).startswith('rpmlib'): continue if not len(q.filter(provides=[require])): if str(require).startswith('('): # rich deps can be only tested by solver if sack is None: sack = dnf.sack.rpmdb_sack(self.base) selector = dnf.selector.Selector(sack) selector.set(provides=str(require)) goal = dnf.goal.Goal(sack) goal.protect_running_kernel = self.base.conf.protect_running_kernel goal.install(select=selector, optional=False) solved = goal.run() # there ase only @system repo in sack, therefore solved is only in case # when rich deps doesn't require any additional package if solved: continue msg = _("{} has missing requires of {}") output_set.add(msg.format( self.base.output.term.bold(pkg), self.base.output.term.bold(require))) for conflict in pkg.conflicts: conflicted = q.filter(provides=[conflict], name=str(conflict).split()[0]) for conflict_pkg in conflicted: msg = '{} has installed conflict "{}": {}' output_set.add(msg.format( self.base.output.term.bold(pkg), self.base.output.term.bold(conflict), self.base.output.term.bold(conflict_pkg))) if self.opts.check_types.intersection({'all', 'duplicates'}): installonly = self.base._get_installonly_query(q) dups = q.duplicated().difference(installonly)._name_dict() for name, pkgs in dups.items(): pkgs.sort() for dup in pkgs[1:]: msg = _("{} is a duplicate with {}").format( self.base.output.term.bold(pkgs[0]), self.base.output.term.bold(dup)) output_set.add(msg) if self.opts.check_types.intersection({'all', 'obsoleted'}): for pkg in q: for obsolete in pkg.obsoletes: obsoleted = q.filter(provides=[obsolete], name=str(obsolete).split()[0]) if len(obsoleted): msg = _("{} is obsoleted by {}").format( self.base.output.term.bold(obsoleted[0]), self.base.output.term.bold(pkg)) output_set.add(msg) if self.opts.check_types.intersection({'all', 'provides'}): for pkg in q: for provide in pkg.provides: if pkg not in q.filter(provides=[provide]): msg = _("{} provides {} but it cannot be found") output_set.add(msg.format( self.base.output.term.bold(pkg), self.base.output.term.bold(provide))) for msg in sorted(output_set): print(msg) if output_set: raise dnf.exceptions.Error( 'Check discovered {} problem(s)'.format(len(output_set))) downgrade.py 0000644 00000004425 15025176534 0007105 0 ustar 00 # downgrade.py # Downgrade CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _ class DowngradeCommand(commands.Command): """A class containing methods needed by the cli to execute the downgrade command. """ aliases = ('downgrade', 'dg') summary = _("Downgrade a package") @staticmethod def set_argparser(parser): parser.add_argument('package', nargs='*', help=_('Package to downgrade'), action=OptionParser.ParseSpecGroupFileCallback) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) if not self.opts.filenames: commands._checkEnabledRepo(self.base) def run(self): file_pkgs = self.base.add_remote_rpms(self.opts.filenames, strict=False, progress=self.base.output.progress) return self.base.downgradePkgs( specs=self.opts.pkg_specs + ['@' + x for x in self.opts.grp_specs], file_pkgs=file_pkgs, strict=self.base.conf.strict) repoquery.py 0000644 00000103414 15025176534 0007164 0 ustar 00 # # Copyright (C) 2014 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals from dnf.i18n import _ from dnf.cli import commands from dnf.cli.option_parser import OptionParser import argparse import datetime import logging import re import sys import dnf import dnf.cli import dnf.exceptions import dnf.subject import dnf.util import hawkey logger = logging.getLogger('dnf') QFORMAT_DEFAULT = '%{name}-%{epoch}:%{version}-%{release}.%{arch}' # matches %[-][dd]{attr} QFORMAT_MATCH = re.compile(r'%(-?\d*?){([:.\w]+?)}') QUERY_TAGS = """\ name, arch, epoch, version, release, reponame (repoid), from_repo, evr, debug_name, source_name, source_debug_name, installtime, buildtime, size, downloadsize, installsize, provides, requires, obsoletes, conflicts, sourcerpm, description, summary, license, url, reason""" OPTS_MAPPING = { 'conflicts': 'conflicts', 'enhances': 'enhances', 'obsoletes': 'obsoletes', 'provides': 'provides', 'recommends': 'recommends', 'requires': 'requires', 'requires-pre': 'requires_pre', 'suggests': 'suggests', 'supplements': 'supplements' } def rpm2py_format(queryformat): """Convert a rpm like QUERYFMT to an python .format() string.""" def fmt_repl(matchobj): fill = matchobj.groups()[0] key = matchobj.groups()[1] if fill: if fill[0] == '-': fill = '>' + fill[1:] else: fill = '<' + fill fill = ':' + fill return '{0.' + key.lower() + fill + "}" def brackets(txt): return txt.replace('{', '{{').replace('}', '}}') queryformat = queryformat.replace("\\n", "\n").replace("\\t", "\t") for key, value in OPTS_MAPPING.items(): queryformat = queryformat.replace(key, value) fmt = "" spos = 0 for item in QFORMAT_MATCH.finditer(queryformat): fmt += brackets(queryformat[spos:item.start()]) fmt += fmt_repl(item) spos = item.end() fmt += brackets(queryformat[spos:]) return fmt class _CommaSplitCallback(OptionParser._SplitCallback): SPLITTER = r'\s*,\s*' class RepoQueryCommand(commands.Command): """A class containing methods needed by the cli to execute the repoquery command. """ nevra_forms = {'repoquery-n': hawkey.FORM_NAME, 'repoquery-na': hawkey.FORM_NA, 'repoquery-nevra': hawkey.FORM_NEVRA} aliases = ('repoquery', 'rq') + tuple(nevra_forms.keys()) summary = _('search for packages matching keyword') @staticmethod def filter_repo_arch(opts, query): """Filter query by repoid and arch options""" if opts.repo: query.filterm(reponame=opts.repo) if opts.arches: query.filterm(arch=opts.arches) return query @staticmethod def set_argparser(parser): parser.add_argument('-a', '--all', dest='queryall', action='store_true', help=_("Query all packages (shorthand for repoquery '*' " "or repoquery without argument)")) parser.add_argument('--show-duplicates', action='store_true', help=_("Query all versions of packages (default)")) parser.add_argument('--arch', '--archlist', dest='arches', default=[], action=_CommaSplitCallback, metavar='[arch]', help=_('show only results from this ARCH')) parser.add_argument('-f', '--file', metavar='FILE', nargs='+', help=_('show only results that owns FILE')) parser.add_argument('--whatconflicts', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that conflict REQ')) parser.add_argument('--whatdepends', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('shows results that requires, suggests, supplements, enhances, ' 'or recommends package provides and files REQ')) parser.add_argument('--whatobsoletes', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that obsolete REQ')) parser.add_argument('--whatprovides', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that provide REQ')) parser.add_argument('--whatrequires', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('shows results that requires package provides and files REQ')) parser.add_argument('--whatrecommends', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that recommend REQ')) parser.add_argument('--whatenhances', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that enhance REQ')) parser.add_argument('--whatsuggests', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that suggest REQ')) parser.add_argument('--whatsupplements', default=[], action=_CommaSplitCallback, metavar='REQ', help=_('show only results that supplement REQ')) whatrequiresform = parser.add_mutually_exclusive_group() whatrequiresform.add_argument("--alldeps", action="store_true", help=_("check non-explicit dependencies (files and Provides); default")) whatrequiresform.add_argument("--exactdeps", action="store_true", help=_('check dependencies exactly as given, opposite of --alldeps')) parser.add_argument("--recursive", action="store_true", help=_( 'used with --whatrequires, and --requires --resolve, query packages recursively.')) parser.add_argument('--deplist', action='store_true', help=_( "show a list of all dependencies and what packages provide them")) parser.add_argument('--resolve', action='store_true', help=_('resolve capabilities to originating package(s)')) parser.add_argument("--tree", action="store_true", help=_('show recursive tree for package(s)')) parser.add_argument('--srpm', action='store_true', help=_('operate on corresponding source RPM')) parser.add_argument("--latest-limit", dest='latest_limit', type=int, help=_('show N latest packages for a given name.arch' ' (or latest but N if N is negative)')) parser.add_argument("--disable-modular-filtering", action="store_true", help=_("list also packages of inactive module streams")) outform = parser.add_mutually_exclusive_group() outform.add_argument('-i', "--info", dest='queryinfo', default=False, action='store_true', help=_('show detailed information about the package')) outform.add_argument('-l', "--list", dest='queryfilelist', default=False, action='store_true', help=_('show list of files in the package')) outform.add_argument('-s', "--source", dest='querysourcerpm', default=False, action='store_true', help=_('show package source RPM name')) outform.add_argument('--changelogs', dest='querychangelogs', default=False, action='store_true', help=_('show changelogs of the package')) outform.add_argument('--qf', "--queryformat", dest='queryformat', default=QFORMAT_DEFAULT, help=_('display format for listing packages: ' '"%%{name} %%{version} ...", ' 'use --querytags to view full tag list')) parser.add_argument('--querytags', action='store_true', help=_('show available tags to use with ' '--queryformat')) outform.add_argument("--nevra", dest='queryformat', const=QFORMAT_DEFAULT, action='store_const', help=_('use name-epoch:version-release.architecture format for ' 'displaying found packages (default)')) outform.add_argument("--nvr", dest='queryformat', const='%{name}-%{version}-%{release}', action='store_const', help=_('use name-version-release format for ' 'displaying found packages ' '(rpm query default)')) outform.add_argument("--envra", dest='queryformat', const='%{epoch}:%{name}-%{version}-%{release}.%{arch}', action='store_const', help=_('use epoch:name-version-release.architecture format for ' 'displaying found packages')) outform.add_argument('--groupmember', action="store_true", help=_( 'Display in which comps groups are presented selected packages')) pkgfilter = parser.add_mutually_exclusive_group() pkgfilter.add_argument("--duplicates", dest='pkgfilter', const='duplicated', action='store_const', help=_('limit the query to installed duplicate ' 'packages')) pkgfilter.add_argument("--duplicated", dest='pkgfilter', const='duplicated', action='store_const', help=argparse.SUPPRESS) pkgfilter.add_argument("--installonly", dest='pkgfilter', const='installonly', action='store_const', help=_('limit the query to installed installonly packages')) pkgfilter.add_argument("--unsatisfied", dest='pkgfilter', const='unsatisfied', action='store_const', help=_('limit the query to installed packages with unsatisfied dependencies')) parser.add_argument('--location', action='store_true', help=_('show a location from where packages can be downloaded')) package_attribute = parser.add_mutually_exclusive_group() help_msgs = { 'conflicts': _('Display capabilities that the package conflicts with.'), 'depends': _('Display capabilities that the package can depend on, enhance, recommend,' ' suggest, and supplement.'), 'enhances': _('Display capabilities that the package can enhance.'), 'provides': _('Display capabilities provided by the package.'), 'recommends': _('Display capabilities that the package recommends.'), 'requires': _('Display capabilities that the package depends on.'), 'requires-pre': _('If the package is not installed display capabilities that it depends on for ' 'running %%pre and %%post scriptlets. If the package is installed display ' 'capabilities that is depends for %%pre, %%post, %%preun and %%postun.'), 'suggests': _('Display capabilities that the package suggests.'), 'supplements': _('Display capabilities that the package can supplement.') } for arg, help_msg in help_msgs.items(): name = '--%s' % arg package_attribute.add_argument(name, dest='packageatr', action='store_const', const=arg, help=help_msg) parser.add_argument('--available', action="store_true", help=_('Display only available packages.')) help_list = { 'installed': _('Display only installed packages.'), 'extras': _('Display only packages that are not present in any of available repositories.'), 'upgrades': _('Display only packages that provide an upgrade for some already installed package.'), 'unneeded': _('Display only packages that can be removed by "{prog} autoremove" ' 'command.').format(prog=dnf.util.MAIN_PROG), 'userinstalled': _('Display only packages that were installed by user.') } list_group = parser.add_mutually_exclusive_group() for list_arg, help_arg in help_list.items(): switch = '--%s' % list_arg list_group.add_argument(switch, dest='list', action='store_const', const=list_arg, help=help_arg) # make --autoremove hidden compatibility alias for --unneeded list_group.add_argument( '--autoremove', dest='list', action='store_const', const="unneeded", help=argparse.SUPPRESS) parser.add_argument('--recent', action="store_true", help=_('Display only recently edited packages')) parser.add_argument('key', nargs='*', metavar="KEY", help=_('the key to search for')) def pre_configure(self): if not self.opts.quiet: self.cli.redirect_logger(stdout=logging.WARNING, stderr=logging.INFO) def configure(self): if not self.opts.quiet: self.cli.redirect_repo_progress() demands = self.cli.demands if self.opts.obsoletes: if self.opts.packageatr: self.cli._option_conflict("--obsoletes", "--" + self.opts.packageatr) else: self.opts.packageatr = "obsoletes" if self.opts.querytags: return if self.opts.resolve and not self.opts.packageatr: raise dnf.cli.CliError( _("Option '--resolve' has to be used together with one of the " "'--conflicts', '--depends', '--enhances', '--provides', '--recommends', " "'--requires', '--requires-pre', '--suggests' or '--supplements' options")) if self.opts.recursive: if self.opts.exactdeps: self.cli._option_conflict("--recursive", "--exactdeps") if not any([self.opts.whatrequires, (self.opts.packageatr == "requires" and self.opts.resolve)]): raise dnf.cli.CliError( _("Option '--recursive' has to be used with '--whatrequires <REQ>' " "(optionally with '--alldeps', but not with '--exactdeps'), or with " "'--requires <REQ> --resolve'")) if self.opts.alldeps or self.opts.exactdeps: if not (self.opts.whatrequires or self.opts.whatdepends): raise dnf.cli.CliError( _("argument {} requires --whatrequires or --whatdepends option".format( '--alldeps' if self.opts.alldeps else '--exactdeps'))) if self.opts.srpm: self.base.repos.enable_source_repos() if (self.opts.list not in ["installed", "userinstalled"] and self.opts.pkgfilter != "installonly") or self.opts.available: demands.available_repos = True demands.sack_activation = True if self.opts.querychangelogs: demands.changelogs = True def build_format_fn(self, opts, pkg): if opts.querychangelogs: out = [] out.append('Changelog for %s' % str(pkg)) for chlog in pkg.changelogs: dt = chlog['timestamp'] out.append('* %s %s\n%s\n' % (dt.strftime("%a %b %d %Y"), dnf.i18n.ucd(chlog['author']), dnf.i18n.ucd(chlog['text']))) return '\n'.join(out) try: po = PackageWrapper(pkg) if opts.queryinfo: return self.base.output.infoOutput(pkg) elif opts.queryfilelist: filelist = po.files if not filelist: print(_('Package {} contains no files').format(pkg), file=sys.stderr) return filelist elif opts.querysourcerpm: return po.sourcerpm else: return rpm2py_format(opts.queryformat).format(po) except AttributeError as e: # catch that the user has specified attributes # there don't exist on the dnf Package object. raise dnf.exceptions.Error(str(e)) def _resolve_nevras(self, nevras, base_query): resolved_nevras_query = self.base.sack.query().filterm(empty=True) for nevra in nevras: resolved_nevras_query = resolved_nevras_query.union(base_query.intersection( dnf.subject.Subject(nevra).get_best_query( self.base.sack, with_provides=False, with_filenames=False ) )) return resolved_nevras_query def _do_recursive_deps(self, query_in, query_select, done=None): done = done if done else query_select query_required = query_in.filter(requires=query_select) query_select = query_required.difference(done) done = query_required.union(done) if query_select: done = self._do_recursive_deps(query_in, query_select, done=done) return done def by_all_deps(self, names, query, all_dep_types=False): # in case of arguments being NEVRAs, resolve them to packages resolved_nevras_query = self._resolve_nevras(names, query) # filter the arguments directly as reldeps depquery = query.filter(requires__glob=names) # filter the resolved NEVRAs as packages depquery = depquery.union(query.filter(requires=resolved_nevras_query)) if all_dep_types: # TODO this is very inefficient, as it resolves the `names` glob to # reldeps four more times, which in a reasonably wide glob like # `dnf repoquery --whatdepends "libdnf*"` can take roughly 50% of # the total execution time. depquery = depquery.union(query.filter(recommends__glob=names)) depquery = depquery.union(query.filter(enhances__glob=names)) depquery = depquery.union(query.filter(supplements__glob=names)) depquery = depquery.union(query.filter(suggests__glob=names)) depquery = depquery.union(query.filter(recommends=resolved_nevras_query)) depquery = depquery.union(query.filter(enhances=resolved_nevras_query)) depquery = depquery.union(query.filter(supplements=resolved_nevras_query)) depquery = depquery.union(query.filter(suggests=resolved_nevras_query)) if self.opts.recursive: depquery = self._do_recursive_deps(query, depquery) return depquery def _get_recursive_providers_query(self, query_in, providers, done=None): done = done if done else self.base.sack.query().filterm(empty=True) t = self.base.sack.query().filterm(empty=True) for pkg in providers.run(): t = t.union(query_in.filter(provides=pkg.requires)) query_select = t.difference(done) if query_select: done = self._get_recursive_providers_query(query_in, query_select, done=t.union(done)) return t.union(done) def _add_add_remote_packages(self): rpmnames = [] remote_packages = [] for key in self.opts.key: schemes = dnf.pycomp.urlparse.urlparse(key)[0] if key.endswith('.rpm'): rpmnames.append(key) elif schemes and schemes in ('http', 'ftp', 'file', 'https'): rpmnames.append(key) if rpmnames: remote_packages = self.base.add_remote_rpms( rpmnames, strict=False, progress=self.base.output.progress) return remote_packages def run(self): if self.opts.querytags: print(QUERY_TAGS) return self.cli._populate_update_security_filter(self.opts) q = self.base.sack.query( flags=hawkey.IGNORE_MODULAR_EXCLUDES if self.opts.disable_modular_filtering else hawkey.APPLY_EXCLUDES ) if self.opts.key: remote_packages = self._add_add_remote_packages() kwark = {'with_provides': False} if self.opts.command in self.nevra_forms: kwark["forms"] = [self.nevra_forms[self.opts.command]] kwark['with_filenames'] = False pkgs = [] query_results = q.filter(empty=True) if remote_packages: query_results = query_results.union( self.base.sack.query().filterm(pkg=remote_packages)) for key in self.opts.key: query_results = query_results.union( dnf.subject.Subject(key, ignore_case=True).get_best_query( self.base.sack, query=q, **kwark)) q = query_results if self.opts.recent: q = q._recent(self.base.conf.recent) if self.opts.available: if self.opts.list and self.opts.list != "installed": print(self.cli.optparser.print_usage()) raise dnf.exceptions.Error(_("argument {}: not allowed with argument {}".format( "--available", "--" + self.opts.list))) elif self.opts.list == "unneeded": q = q._unneeded(self.base.history.swdb) elif self.opts.list and self.opts.list != 'userinstalled': q = getattr(q, self.opts.list)() if self.opts.pkgfilter == "duplicated": installonly = self.base._get_installonly_query(q) q = q.difference(installonly).duplicated() elif self.opts.pkgfilter == "installonly": q = self.base._get_installonly_query(q) elif self.opts.pkgfilter == "unsatisfied": rpmdb = dnf.sack.rpmdb_sack(self.base) rpmdb._configure(self.base.conf.installonlypkgs, self.base.conf.installonly_limit) goal = dnf.goal.Goal(rpmdb) goal.protect_running_kernel = False solved = goal.run(verify=True) if not solved: print(dnf.util._format_resolve_problems(goal.problem_rules())) return elif not self.opts.list: # do not show packages from @System repo q = q.available() # filter repo and arch q = self.filter_repo_arch(self.opts, q) orquery = q if self.opts.file: q.filterm(file__glob=self.opts.file) if self.opts.whatconflicts: rels = q.filter(conflicts__glob=self.opts.whatconflicts) q = rels.union(q.filter(conflicts=self._resolve_nevras(self.opts.whatconflicts, q))) if self.opts.whatobsoletes: q.filterm(obsoletes=self.opts.whatobsoletes) if self.opts.whatprovides: query_for_provide = q.filter(provides__glob=self.opts.whatprovides) if query_for_provide: q = query_for_provide else: q.filterm(file__glob=self.opts.whatprovides) if self.opts.whatrequires: if (self.opts.exactdeps): q.filterm(requires__glob=self.opts.whatrequires) else: q = self.by_all_deps(self.opts.whatrequires, q) if self.opts.whatdepends: if (self.opts.exactdeps): dependsquery = q.filter(requires__glob=self.opts.whatdepends) dependsquery = dependsquery.union(q.filter(recommends__glob=self.opts.whatdepends)) dependsquery = dependsquery.union(q.filter(enhances__glob=self.opts.whatdepends)) dependsquery = dependsquery.union(q.filter(supplements__glob=self.opts.whatdepends)) q = dependsquery.union(q.filter(suggests__glob=self.opts.whatdepends)) else: q = self.by_all_deps(self.opts.whatdepends, q, True) if self.opts.whatrecommends: rels = q.filter(recommends__glob=self.opts.whatrecommends) q = rels.union(q.filter(recommends=self._resolve_nevras(self.opts.whatrecommends, q))) if self.opts.whatenhances: rels = q.filter(enhances__glob=self.opts.whatenhances) q = rels.union(q.filter(enhances=self._resolve_nevras(self.opts.whatenhances, q))) if self.opts.whatsupplements: rels = q.filter(supplements__glob=self.opts.whatsupplements) q = rels.union(q.filter(supplements=self._resolve_nevras(self.opts.whatsupplements, q))) if self.opts.whatsuggests: rels = q.filter(suggests__glob=self.opts.whatsuggests) q = rels.union(q.filter(suggests=self._resolve_nevras(self.opts.whatsuggests, q))) if self.opts.latest_limit: q = q.latest(self.opts.latest_limit) # reduce a query to security upgrades if they are specified q = self.base._merge_update_filters(q, warning=False) if self.opts.srpm: pkg_list = [] for pkg in q: srcname = pkg.source_name if srcname is not None: tmp_query = self.base.sack.query().filterm(name=srcname, evr=pkg.evr, arch='src') pkg_list += tmp_query.run() q = self.base.sack.query().filterm(pkg=pkg_list) if self.opts.tree: if not self.opts.whatrequires and self.opts.packageatr not in ( 'conflicts', 'enhances', 'obsoletes', 'provides', 'recommends', 'requires', 'suggests', 'supplements'): raise dnf.exceptions.Error( _("No valid switch specified\nusage: {prog} repoquery [--conflicts|" "--enhances|--obsoletes|--provides|--recommends|--requires|" "--suggest|--supplements|--whatrequires] [key] [--tree]\n\n" "description:\n For the given packages print a tree of the " "packages.").format(prog=dnf.util.MAIN_PROG)) self.tree_seed(q, orquery, self.opts) return pkgs = set() if self.opts.packageatr: rels = set() for pkg in q.run(): if self.opts.list != 'userinstalled' or self.base.history.user_installed(pkg): if self.opts.packageatr == 'depends': rels.update(pkg.requires + pkg.enhances + pkg.suggests + pkg.supplements + pkg.recommends) else: rels.update(getattr(pkg, OPTS_MAPPING[self.opts.packageatr])) if self.opts.resolve: # find the providing packages and show them if self.opts.list == "installed": query = self.filter_repo_arch(self.opts, self.base.sack.query()) else: query = self.filter_repo_arch(self.opts, self.base.sack.query().available()) providers = query.filter(provides=rels) if self.opts.recursive: providers = providers.union( self._get_recursive_providers_query(query, providers)) pkgs = set() for pkg in providers.latest().run(): pkgs.add(self.build_format_fn(self.opts, pkg)) else: pkgs.update(str(rel) for rel in rels) elif self.opts.location: for pkg in q.run(): location = pkg.remote_location() if location is not None: pkgs.add(location) elif self.opts.deplist: pkgs = [] for pkg in sorted(set(q.run())): if self.opts.list != 'userinstalled' or self.base.history.user_installed(pkg): deplist_output = [] deplist_output.append('package: ' + str(pkg)) for req in sorted([str(req) for req in pkg.requires]): deplist_output.append(' dependency: ' + req) subject = dnf.subject.Subject(req) query = subject.get_best_query(self.base.sack) query = self.filter_repo_arch( self.opts, query.available()) if not self.opts.verbose: query = query.latest() for provider in query.run(): deplist_output.append(' provider: ' + str(provider)) pkgs.append('\n'.join(deplist_output)) if pkgs: print('\n\n'.join(pkgs)) return elif self.opts.groupmember: self._group_member_report(q) return else: for pkg in q.run(): if self.opts.list != 'userinstalled' or self.base.history.user_installed(pkg): pkgs.add(self.build_format_fn(self.opts, pkg)) if pkgs: if self.opts.queryinfo: print("\n\n".join(sorted(pkgs))) else: print("\n".join(sorted(pkgs))) def _group_member_report(self, query): package_conf_dict = {} for group in self.base.comps.groups: package_conf_dict[group.id] = set([pkg.name for pkg in group.packages_iter()]) group_package_dict = {} pkg_not_in_group = [] for pkg in query.run(): group_id_list = [] for group_id, package_name_set in package_conf_dict.items(): if pkg.name in package_name_set: group_id_list.append(group_id) if group_id_list: group_package_dict.setdefault( '$'.join(sorted(group_id_list)), []).append(str(pkg)) else: pkg_not_in_group.append(str(pkg)) output = [] for key, package_list in sorted(group_package_dict.items()): output.append( '\n'.join(sorted(package_list) + sorted([' @' + id for id in key.split('$')]))) output.append('\n'.join(sorted(pkg_not_in_group))) if output: print('\n'.join(output)) def grow_tree(self, level, pkg, opts): pkg_string = self.build_format_fn(opts, pkg) if level == -1: print(pkg_string) return spacing = " " for x in range(0, level): spacing += "| " requires = [] for requirepkg in pkg.requires: requires.append(str(requirepkg)) reqstr = "[" + str(len(requires)) + ": " + ", ".join(requires) + "]" print(spacing + r"\_ " + pkg_string + " " + reqstr) def tree_seed(self, query, aquery, opts, level=-1, usedpkgs=None): for pkg in sorted(set(query.run()), key=lambda p: p.name): usedpkgs = set() if usedpkgs is None or level == -1 else usedpkgs if pkg.name.startswith("rpmlib") or pkg.name.startswith("solvable"): return self.grow_tree(level, pkg, opts) if pkg not in usedpkgs: usedpkgs.add(pkg) if opts.packageatr: strpkg = getattr(pkg, opts.packageatr) ar = {} for name in set(strpkg): pkgquery = self.base.sack.query().filterm(provides=name) for querypkg in pkgquery: ar[querypkg.name + "." + querypkg.arch] = querypkg pkgquery = self.base.sack.query().filterm(pkg=list(ar.values())) else: pkgquery = self.by_all_deps((pkg.name, ), aquery) if opts.alldeps \ else aquery.filter(requires__glob=pkg.name) self.tree_seed(pkgquery, aquery, opts, level + 1, usedpkgs) class PackageWrapper(object): """Wrapper for dnf.package.Package, so we can control formatting.""" def __init__(self, pkg): self._pkg = pkg def __getattr__(self, attr): atr = getattr(self._pkg, attr) if atr is None: return "(none)" if isinstance(atr, list): return '\n'.join(sorted({dnf.i18n.ucd(reldep) for reldep in atr})) return dnf.i18n.ucd(atr) @staticmethod def _get_timestamp(timestamp): if timestamp > 0: dt = datetime.datetime.utcfromtimestamp(timestamp) return dt.strftime("%Y-%m-%d %H:%M") else: return '' @property def buildtime(self): return self._get_timestamp(self._pkg.buildtime) @property def installtime(self): return self._get_timestamp(self._pkg.installtime) shell.py 0000644 00000023154 15025176534 0006242 0 ustar 00 # shell.py # Shell CLI command. # # Copyright (C) 2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from dnf.cli import commands from dnf.i18n import _, ucd import dnf.util import cmd import copy import dnf import logging import shlex import sys logger = logging.getLogger('dnf') # only demands we'd like to override class ShellDemandSheet(object): available_repos = True resolving = True root_user = True sack_activation = True class ShellCommand(commands.Command, cmd.Cmd): aliases = ('shell', 'sh') summary = _('run an interactive {prog} shell').format(prog=dnf.util.MAIN_PROG_UPPER) MAPPING = {'repo': 'repo', 'repository': 'repo', 'exit': 'quit', 'quit': 'quit', 'run': 'ts_run', 'ts': 'transaction', 'transaction': 'transaction', 'config': 'config', 'resolvedep': 'resolve', 'help': 'help' } def __init__(self, cli): commands.Command.__init__(self, cli) cmd.Cmd.__init__(self) self.prompt = '> ' @staticmethod def set_argparser(parser): parser.add_argument('script', nargs='?', metavar=_('SCRIPT'), help=_('Script to run in {prog} shell').format( prog=dnf.util.MAIN_PROG_UPPER)) def configure(self): # append to ShellDemandSheet missing demands from # dnf.cli.demand.DemandSheet with their default values. default_demands = self.cli.demands self.cli.demands = ShellDemandSheet() for attr in dir(default_demands): if attr.startswith('__'): continue try: getattr(self.cli.demands, attr) except AttributeError: setattr(self.cli.demands, attr, getattr(default_demands, attr)) def run(self): if self.opts.script: self._run_script(self.opts.script) else: self.cmdloop() def _clean(self): self.base._finalize_base() self.base._transaction = None self.base.fill_sack() def onecmd(self, line): if not line or line == '\n': return if line == 'EOF': line = 'quit' try: s_line = shlex.split(line) except: self._help() return # reset option parser before each command, keep usage information self.cli.optparser.__init__(reset_usage=False) opts = self.cli.optparser.parse_main_args(s_line) # Disable shell recursion. if opts.command == 'shell': return if opts.command in self.MAPPING: getattr(self, '_' + self.MAPPING[opts.command])(s_line[1::]) else: cmd_cls = self.cli.cli_commands.get(opts.command) if cmd_cls is not None: cmd = cmd_cls(self.cli) try: opts = self.cli.optparser.parse_command_args(cmd, s_line) except SystemExit: # argparse.ArgumentParser prints usage information and executes # sys.exit() on problems with parsing command line arguments return try: cmd.cli.demands = copy.deepcopy(self.cli.demands) cmd.configure() cmd.run() except dnf.exceptions.Error as e: logger.error(_("Error:") + " " + ucd(e)) return else: self._help() def _config(self, args=None): def print_or_set(key, val, conf): if val: setattr(conf, key, val) else: try: print('{}: {}'.format(key, getattr(conf, str(key)))) except: logger.warning(_('Unsupported key value.')) if not args or len(args) > 2: self._help('config') return key = args[0] val = args[1] if len(args) == 2 else None period = key.find('.') if period != -1: repo_name = key[:period] key = key[period+1:] repos = self.base.repos.get_matching(repo_name) for repo in repos: print_or_set(key, val, repo) if not repos: logger.warning(_('Could not find repository: %s'), repo_name) else: print_or_set(key, val, self.base.conf) def _help(self, args=None): """Output help information. :param args: the command to output help information about. If *args* is an empty, general help will be output. """ arg = args[0] if isinstance(args, list) and len(args) > 0 else args msg = None if arg: if arg == 'config': msg = _("""{} arg [value] arg: debuglevel, errorlevel, obsoletes, gpgcheck, assumeyes, exclude, repo_id.gpgcheck, repo_id.exclude If no value is given it prints the current value. If value is given it sets that value.""").format(arg) elif arg == 'help': msg = _("""{} [command] print help""").format(arg) elif arg in ['repo', 'repository']: msg = _("""{} arg [option] list: lists repositories and their status. option = [all | id | glob] enable: enable repositories. option = repository id disable: disable repositories. option = repository id""").format(arg) elif arg == 'resolvedep': msg = _("""{} resolve the transaction set""").format(arg) elif arg in ['transaction', 'ts']: msg = _("""{} arg list: lists the contents of the transaction reset: reset (zero-out) the transaction run: run the transaction""").format(arg) elif arg == 'run': msg = _("""{} run the transaction""").format(arg) elif arg in ['exit', 'quit']: msg = _("""{} exit the shell""").format(arg) if not msg: self.cli.optparser.print_help() msg = _("""Shell specific arguments: config set config options help print help repository (or repo) enable, disable or list repositories resolvedep resolve the transaction set transaction (or ts) list, reset or run the transaction set run resolve and run the transaction set exit (or quit) exit the shell""") print('\n' + msg) def _repo(self, args=None): cmd = args[0] if args else None if cmd in ['list', None]: self.onecmd('repolist ' + ' '.join(args[1:])) elif cmd in ['enable', 'disable']: repos = self.cli.base.repos fill_sack = False for repo in args[1::]: r = repos.get_matching(repo) if r: getattr(r, cmd)() fill_sack = True else: logger.critical(_("Error:") + " " + _("Unknown repo: '%s'"), self.base.output.term.bold(repo)) if fill_sack: self.base.fill_sack() # reset base._comps, as it has changed due to changing the repos self.base._comps = None else: self._help('repo') def _resolve(self, args=None): try: self.cli.base.resolve(self.cli.demands.allow_erasing) except dnf.exceptions.DepsolveError as e: print(e) def _run_script(self, file): try: with open(file, 'r') as fd: lines = fd.readlines() for line in lines: if not line.startswith('#'): self.onecmd(line) except IOError: logger.info(_('Error: Cannot open %s for reading'), self.base.output.term.bold(file)) sys.exit(1) def _transaction(self, args=None): cmd = args[0] if args else None if cmd == 'reset': self._clean() return self._resolve() if cmd in ['list', None]: if self.base._transaction: out = self.base.output.list_transaction(self.base._transaction) logger.info(out) elif cmd == 'run': try: self.base.do_transaction() except dnf.exceptions.Error as e: logger.error(_("Error:") + " " + ucd(e)) else: logger.info(_("Complete!")) self._clean() else: self._help('transaction') def _ts_run(self, args=None): self._transaction(['run']) def _quit(self, args=None): logger.info(_('Leaving Shell')) sys.exit(0) mark.py 0000644 00000006716 15025176534 0006072 0 ustar 00 # mark.py # Mark CLI command. # # Copyright (C) 2015-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import print_function from __future__ import unicode_literals import libdnf.transaction from dnf.i18n import _ from dnf.cli import commands import dnf import functools import logging logger = logging.getLogger("dnf") class MarkCommand(commands.Command): aliases = ('mark',) summary = _('mark or unmark installed packages as installed by user.') @staticmethod def set_argparser(parser): parser.add_argument('mark', nargs=1, choices=['install', 'remove', 'group'], help=_("install: mark as installed by user\n" "remove: unmark as installed by user\n" "group: mark as installed by group")) parser.add_argument('package', nargs='+', metavar="PACKAGE", help=_("Package specification")) def _mark_install(self, pkg): self.base.history.set_reason(pkg, libdnf.transaction.TransactionItemReason_USER) logger.info(_('%s marked as user installed.'), str(pkg)) def _mark_remove(self, pkg): self.base.history.set_reason(pkg, libdnf.transaction.TransactionItemReason_DEPENDENCY) logger.info(_('%s unmarked as user installed.'), str(pkg)) def _mark_group(self, pkg): self.base.history.set_reason(pkg, libdnf.transaction.TransactionItemReason_GROUP) logger.info(_('%s marked as group installed.'), str(pkg)) def configure(self): demands = self.cli.demands demands.sack_activation = True demands.root_user = True demands.available_repos = False demands.resolving = False def run(self): cmd = self.opts.mark[0] pkgs = self.opts.package mark_func = functools.partial(getattr(self, '_mark_' + cmd)) notfound = [] for pkg in pkgs: subj = dnf.subject.Subject(pkg) q = subj.get_best_query(self.base.sack) for pkg in q: mark_func(pkg) if len(q) == 0: notfound.append(pkg) if notfound: logger.error(_('Error:')) for pkg in notfound: logger.error(_('Package %s is not installed.'), pkg) raise dnf.cli.CliError old = self.base.history.last() if old is None: rpmdb_version = self.base._ts.dbCookie() else: rpmdb_version = old.end_rpmdb_version self.base.history.beg(rpmdb_version, [], []) self.base.history.end(rpmdb_version) clean.py 0000644 00000010556 15025176534 0006217 0 ustar 00 # clean.py # Clean CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.i18n import _, P_ from dnf.yum import misc import dnf.cli import dnf.exceptions import dnf.lock import dnf.logging import dnf.repo import logging import os import re import time logger = logging.getLogger("dnf") # Dict mapping cmdline arguments to actual data types to be cleaned up _CACHE_TYPES = { 'metadata': ['metadata', 'dbcache', 'expire-cache'], 'packages': ['packages'], 'dbcache': ['dbcache'], 'expire-cache': ['expire-cache'], 'all': ['metadata', 'packages', 'dbcache'], } def _tree(dirpath): """Traverse dirpath recursively and yield relative filenames.""" for root, dirs, files in os.walk(dirpath): base = os.path.relpath(root, dirpath) for f in files: path = os.path.join(base, f) yield os.path.normpath(path) def _filter(files, patterns): """Yield those filenames that match any of the patterns.""" return (f for f in files for p in patterns if re.match(p, f)) def _clean(dirpath, files): """Remove the given filenames from dirpath.""" count = 0 for f in files: path = os.path.join(dirpath, f) logger.log(dnf.logging.DDEBUG, _('Removing file %s'), path) misc.unlink_f(path) count += 1 return count def _cached_repos(files): """Return the repo IDs that have some cached metadata around.""" metapat = dnf.repo.CACHE_FILES['metadata'] matches = (re.match(metapat, f) for f in files) return set(m.group('repoid') for m in matches if m) class CleanCommand(commands.Command): """A class containing methods needed by the cli to execute the clean command. """ aliases = ('clean',) summary = _('remove cached data') @staticmethod def set_argparser(parser): parser.add_argument('type', nargs='+', choices=_CACHE_TYPES.keys(), help=_('Metadata type to clean')) def run(self): cachedir = self.base.conf.cachedir md_lock = dnf.lock.build_metadata_lock(cachedir, True) download_lock = dnf.lock.build_download_lock(cachedir, True) rpmdb_lock = dnf.lock.build_rpmdb_lock(self.base.conf.persistdir, True) while True: try: with md_lock, download_lock, rpmdb_lock: types = set(t for c in self.opts.type for t in _CACHE_TYPES[c]) files = list(_tree(cachedir)) logger.debug(_('Cleaning data: ' + ' '.join(types))) if 'expire-cache' in types: expired = _cached_repos(files) self.base._repo_persistor.expired_to_add.update(expired) types.remove('expire-cache') logger.info(_('Cache was expired')) patterns = [dnf.repo.CACHE_FILES[t] for t in types] count = _clean(cachedir, _filter(files, patterns)) logger.info(P_('%d file removed', '%d files removed', count) % count) return except dnf.exceptions.LockError as e: if not self.base.conf.exit_on_lock: msg = _('Waiting for process with pid %d to finish.') % (e.pid) logger.info(msg) time.sleep(3) else: raise e makecache.py 0000644 00000003555 15025176534 0007037 0 ustar 00 # makecache.py # Makecache CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.i18n import _ import argparse import dnf.cli import dnf.exceptions import dnf.util import logging logger = logging.getLogger("dnf") class MakeCacheCommand(commands.Command): aliases = ('makecache', 'mc') summary = _('generate the metadata cache') @staticmethod def set_argparser(parser): parser.add_argument('--timer', action='store_true', dest="timer_opt") # compatibility with dnf < 2.0 parser.add_argument('timer', nargs='?', choices=['timer'], metavar='timer', help=argparse.SUPPRESS) def run(self): timer = self.opts.timer is not None or self.opts.timer_opt msg = _("Making cache files for all metadata files.") logger.debug(msg) return self.base.update_cache(timer) autoremove.py 0000644 00000005746 15025176534 0007330 0 ustar 00 # autoremove.py # Autoremove CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _ import dnf.exceptions import hawkey import logging logger = logging.getLogger("dnf") class AutoremoveCommand(commands.Command): nevra_forms = {'autoremove-n': hawkey.FORM_NAME, 'autoremove-na': hawkey.FORM_NA, 'autoremove-nevra': hawkey.FORM_NEVRA} aliases = ('autoremove',) + tuple(nevra_forms.keys()) summary = _('remove all unneeded packages that were originally installed ' 'as dependencies') @staticmethod def set_argparser(parser): parser.add_argument('packages', nargs='*', help=_('Package to remove'), action=OptionParser.ParseSpecGroupFileCallback, metavar=_('PACKAGE')) def configure(self): demands = self.cli.demands demands.resolving = True demands.root_user = True demands.sack_activation = True if any([self.opts.grp_specs, self.opts.pkg_specs, self.opts.filenames]): self.base.conf.clean_requirements_on_remove = True demands.allow_erasing = True # disable all available repos to delete whole dependency tree # instead of replacing removable package with available packages demands.available_repos = False else: demands.available_repos = True demands.fresh_metadata = False def run(self): if any([self.opts.grp_specs, self.opts.pkg_specs, self.opts.filenames]): forms = [] if self.opts.command in self.nevra_forms: forms = [self.nevra_forms[self.opts.command]] self.base.autoremove(forms, self.opts.pkg_specs, self.opts.grp_specs, self.opts.filenames) else: self.base.autoremove() repolist.py 0000644 00000031172 15025176534 0006773 0 ustar 00 # repolist.py # repolist CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals from dnf.cli import commands from dnf.i18n import _, ucd, fill_exact_width, exact_width from dnf.cli.option_parser import OptionParser import dnf.cli.format import dnf.pycomp import dnf.util import fnmatch import hawkey import logging import operator logger = logging.getLogger('dnf') def _expire_str(repo, md): last = dnf.util.normalize_time(repo._repo.getTimestamp()) if md else _("unknown") if repo.metadata_expire <= -1: return _("Never (last: %s)") % last elif not repo.metadata_expire: return _("Instant (last: %s)") % last else: num = _num2ui_num(repo.metadata_expire) return _("%s second(s) (last: %s)") % (num, last) def _num2ui_num(num): return ucd(dnf.pycomp.format("%d", num, True)) def _repo_match(repo, patterns): rid = repo.id.lower() rnm = repo.name.lower() for pat in patterns: if fnmatch.fnmatch(rid, pat): return True if fnmatch.fnmatch(rnm, pat): return True return False def _repo_size(sack, repo): ret = 0 for pkg in sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm(reponame__eq=repo.id): ret += pkg._size return dnf.cli.format.format_number(ret) class RepoListCommand(commands.Command): """A class containing methods needed by the cli to execute the repolist command. """ aliases = ('repolist', 'repoinfo') summary = _('display the configured software repositories') @staticmethod def set_argparser(parser): repolimit = parser.add_mutually_exclusive_group() repolimit.add_argument('--all', dest='_repos_action', action='store_const', const='all', default=None, help=_("show all repos")) repolimit.add_argument('--enabled', dest='_repos_action', action='store_const', const='enabled', help=_("show enabled repos (default)")) repolimit.add_argument('--disabled', dest='_repos_action', action='store_const', const='disabled', help=_("show disabled repos")) parser.add_argument('repos', nargs='*', default='enabled-default', metavar="REPOSITORY", choices=['all', 'enabled', 'disabled'], action=OptionParser.PkgNarrowCallback, help=_("Repository specification")) def pre_configure(self): if not self.opts.quiet: self.cli.redirect_logger(stdout=logging.WARNING, stderr=logging.INFO) def configure(self): if not self.opts.quiet: self.cli.redirect_repo_progress() demands = self.cli.demands if self.base.conf.verbose or self.opts.command == 'repoinfo': demands.available_repos = True demands.sack_activation = True if self.opts._repos_action: self.opts.repos_action = self.opts._repos_action def run(self): arg = self.opts.repos_action extcmds = [x.lower() for x in self.opts.repos] verbose = self.base.conf.verbose repos = list(self.base.repos.values()) repos.sort(key=operator.attrgetter('id')) term = self.output.term on_ehibeg = term.FG_COLOR['green'] + term.MODE['bold'] on_dhibeg = term.FG_COLOR['red'] on_hiend = term.MODE['normal'] tot_num = 0 cols = [] if not repos: logger.warning(_('No repositories available')) return include_status = arg == 'all' or (arg == 'enabled-default' and extcmds) repoinfo_output = [] for repo in repos: if len(extcmds) and not _repo_match(repo, extcmds): continue (ehibeg, dhibeg, hiend) = '', '', '' ui_enabled = '' ui_endis_wid = 0 ui_excludes_num = '' if include_status: (ehibeg, dhibeg, hiend) = (on_ehibeg, on_dhibeg, on_hiend) if repo.enabled: enabled = True if arg == 'disabled': continue if include_status or verbose or self.opts.command == 'repoinfo': ui_enabled = ehibeg + _('enabled') + hiend ui_endis_wid = exact_width(_('enabled')) if verbose or self.opts.command == 'repoinfo': ui_size = _repo_size(self.base.sack, repo) else: enabled = False if arg == 'enabled' or (arg == 'enabled-default' and not extcmds): continue ui_enabled = dhibeg + _('disabled') + hiend ui_endis_wid = exact_width(_('disabled')) if not (verbose or self.opts.command == 'repoinfo'): rid = ucd(repo.id) cols.append((rid, repo.name, (ui_enabled, ui_endis_wid))) else: if enabled: md = repo.metadata else: md = None out = [self.output.fmtKeyValFill(_("Repo-id : "), repo.id), self.output.fmtKeyValFill(_("Repo-name : "), repo.name)] if include_status: out += [self.output.fmtKeyValFill(_("Repo-status : "), ui_enabled)] if md and repo._repo.getRevision(): out += [self.output.fmtKeyValFill(_("Repo-revision : "), repo._repo.getRevision())] if md and repo._repo.getContentTags(): tags = repo._repo.getContentTags() out += [self.output.fmtKeyValFill(_("Repo-tags : "), ", ".join(sorted(tags)))] if md and repo._repo.getDistroTags(): distroTagsDict = {k: v for (k, v) in repo._repo.getDistroTags()} for (distro, tags) in distroTagsDict.items(): out += [self.output.fmtKeyValFill( _("Repo-distro-tags : "), "[%s]: %s" % (distro, ", ".join(sorted(tags))))] if md: num = len(self.base.sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm( reponame__eq=repo.id)) num_available = len(self.base.sack.query().filterm(reponame__eq=repo.id)) ui_num = _num2ui_num(num) ui_num_available = _num2ui_num(num_available) tot_num += num out += [ self.output.fmtKeyValFill( _("Repo-updated : "), dnf.util.normalize_time(repo._repo.getMaxTimestamp())), self.output.fmtKeyValFill(_("Repo-pkgs : "), ui_num), self.output.fmtKeyValFill(_("Repo-available-pkgs: "), ui_num_available), self.output.fmtKeyValFill(_("Repo-size : "), ui_size)] if repo.metalink: out += [self.output.fmtKeyValFill(_("Repo-metalink : "), repo.metalink)] if enabled: ts = repo._repo.getTimestamp() out += [self.output.fmtKeyValFill( _(" Updated : "), dnf.util.normalize_time(ts))] elif repo.mirrorlist: out += [self.output.fmtKeyValFill(_("Repo-mirrors : "), repo.mirrorlist)] baseurls = repo.baseurl if baseurls: out += [self.output.fmtKeyValFill(_("Repo-baseurl : "), ", ".join(baseurls))] elif enabled: mirrors = repo._repo.getMirrors() if mirrors: url = "%s (%d more)" % (mirrors[0], len(mirrors) - 1) out += [self.output.fmtKeyValFill(_("Repo-baseurl : "), url)] expire = _expire_str(repo, md) out += [self.output.fmtKeyValFill(_("Repo-expire : "), expire)] if repo.excludepkgs: # TRANSLATORS: Packages that are excluded - their names like (dnf systemd) out += [self.output.fmtKeyValFill(_("Repo-exclude : "), ", ".join(repo.excludepkgs))] if repo.includepkgs: out += [self.output.fmtKeyValFill(_("Repo-include : "), ", ".join(repo.includepkgs))] if ui_excludes_num: # TRANSLATORS: Number of packages that where excluded (5) out += [self.output.fmtKeyValFill(_("Repo-excluded : "), ui_excludes_num)] if repo.repofile: out += [self.output.fmtKeyValFill(_("Repo-filename : "), repo.repofile)] repoinfo_output.append("\n".join(map(ucd, out))) if repoinfo_output: print("\n\n".join(repoinfo_output)) if not verbose and cols: # Work out the first (id) and last (enabled/disabled/count), # then chop the middle (name)... id_len = exact_width(_('repo id')) nm_len = 0 st_len = 0 for (rid, rname, (ui_enabled, ui_endis_wid)) in cols: if id_len < exact_width(rid): id_len = exact_width(rid) if nm_len < exact_width(rname): nm_len = exact_width(rname) if st_len < ui_endis_wid: st_len = ui_endis_wid # Need this as well as above for: fill_exact_width() if include_status: if exact_width(_('status')) > st_len: left = term.columns - (id_len + len(_('status')) + 2) else: left = term.columns - (id_len + st_len + 2) else: # Don't output a status column. left = term.columns - (id_len + 1) if left < nm_len: # Name gets chopped nm_len = left else: # Share the extra... left -= nm_len id_len += left // 2 nm_len += left - (left // 2) txt_rid = fill_exact_width(_('repo id'), id_len) if include_status: txt_rnam = fill_exact_width(_('repo name'), nm_len, nm_len) else: txt_rnam = _('repo name') if not include_status: # Don't output a status column. print("%s %s" % (txt_rid, txt_rnam)) else: print("%s %s %s" % (txt_rid, txt_rnam, _('status'))) for (rid, rname, (ui_enabled, ui_endis_wid)) in cols: if not include_status: # Don't output a status column. print("%s %s" % (fill_exact_width(rid, id_len), rname)) continue print("%s %s %s" % (fill_exact_width(rid, id_len), fill_exact_width(rname, nm_len, nm_len), ui_enabled)) if verbose or self.opts.command == 'repoinfo': msg = _('Total packages: {}') print(msg.format(_num2ui_num(tot_num))) install.py 0000644 00000017123 15025176534 0006600 0 ustar 00 # install.py # Install CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # from __future__ import absolute_import from __future__ import unicode_literals import logging from itertools import chain import hawkey import dnf.exceptions from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _ logger = logging.getLogger('dnf') class InstallCommand(commands.Command): """A class containing methods needed by the cli to execute the install command. """ nevra_forms = {'install-n': hawkey.FORM_NAME, 'install-na': hawkey.FORM_NA, 'install-nevra': hawkey.FORM_NEVRA} alternatives_provide = 'alternative-for({})' aliases = ('install', 'localinstall', 'in') + tuple(nevra_forms.keys()) summary = _('install a package or packages on your system') @staticmethod def set_argparser(parser): parser.add_argument('package', nargs='+', metavar=_('PACKAGE'), action=OptionParser.ParseSpecGroupFileCallback, help=_('Package to install')) def configure(self): """Verify that conditions are met so that this command can run. That there are enabled repositories with gpg keys, and that this command is called with appropriate arguments. """ demands = self.cli.demands demands.sack_activation = True demands.available_repos = True demands.resolving = True demands.root_user = True commands._checkGPGKey(self.base, self.cli) if not self.opts.filenames: commands._checkEnabledRepo(self.base) def run(self): err_pkgs = [] errs = [] error_module_specs = [] nevra_forms = self._get_nevra_forms_from_command() self.cli._populate_update_security_filter(self.opts) if self.opts.command == 'localinstall' and (self.opts.grp_specs or self.opts.pkg_specs): self._log_not_valid_rpm_file_paths(self.opts.grp_specs) if self.base.conf.strict: raise dnf.exceptions.Error(_('Nothing to do.')) skipped_grp_specs = [] if self.opts.grp_specs and self.opts.command != 'localinstall': if dnf.base.WITH_MODULES: try: module_base = dnf.module.module_base.ModuleBase(self.base) module_base.install(self.opts.grp_specs, strict=self.base.conf.strict) except dnf.exceptions.MarkingErrors as e: if e.no_match_group_specs: for e_spec in e.no_match_group_specs: skipped_grp_specs.append(e_spec) if e.error_group_specs: for e_spec in e.error_group_specs: error_module_specs.append("@" + e_spec) module_depsolv_errors = e.module_depsolv_errors if module_depsolv_errors: logger.error(dnf.module.module_base.format_modular_solver_errors( module_depsolv_errors[0])) else: skipped_grp_specs = self.opts.grp_specs if self.opts.filenames and nevra_forms: self._inform_not_a_valid_combination(self.opts.filenames) if self.base.conf.strict: raise dnf.exceptions.Error(_('Nothing to do.')) else: err_pkgs = self._install_files() if skipped_grp_specs and nevra_forms: self._inform_not_a_valid_combination(skipped_grp_specs) if self.base.conf.strict: raise dnf.exceptions.Error(_('Nothing to do.')) elif skipped_grp_specs and self.opts.command != 'localinstall': self._install_groups(skipped_grp_specs) if self.opts.command != 'localinstall': errs = self._install_packages(nevra_forms) if (len(errs) != 0 or len(err_pkgs) != 0 or error_module_specs) and self.base.conf.strict: raise dnf.exceptions.PackagesNotAvailableError(_("Unable to find a match"), pkg_spec=' '.join(errs), packages=err_pkgs) def _get_nevra_forms_from_command(self): if self.opts.command in self.nevra_forms: return [self.nevra_forms[self.opts.command]] else: return [] def _log_not_valid_rpm_file_paths(self, grp_specs): group_names = map(lambda g: '@' + g, grp_specs) for pkg in chain(self.opts.pkg_specs, group_names): msg = _('Not a valid rpm file path: %s') logger.info(msg, self.base.output.term.bold(pkg)) def _inform_not_a_valid_combination(self, forms): for form in forms: msg = _('Not a valid form: %s') logger.warning(msg, self.base.output.term.bold(form)) def _install_files(self): err_pkgs = [] strict = self.base.conf.strict for pkg in self.base.add_remote_rpms(self.opts.filenames, strict=strict, progress=self.base.output.progress): try: self.base.package_install(pkg, strict=strict) except dnf.exceptions.MarkingError: msg = _('No match for argument: %s') logger.info(msg, self.base.output.term.bold(pkg.location)) err_pkgs.append(pkg) return err_pkgs def _install_groups(self, grp_specs): try: self.base.env_group_install(grp_specs, tuple(self.base.conf.group_package_types), strict=self.base.conf.strict) except dnf.exceptions.Error: if self.base.conf.strict: raise def _report_alternatives(self, pkg_spec): query = self.base.sack.query().filterm( provides=self.alternatives_provide.format(pkg_spec)) if query: msg = _('There are following alternatives for "{0}": {1}') logger.info(msg.format( pkg_spec, ', '.join(sorted(set([alt.name for alt in query]))))) def _install_packages(self, nevra_forms): errs = [] strict = self.base.conf.strict for pkg_spec in self.opts.pkg_specs: try: self.base.install(pkg_spec, strict=strict, forms=nevra_forms) except dnf.exceptions.MarkingError as e: msg = '{}: {}'.format(e.value, self.base.output.term.bold(pkg_spec)) logger.info(msg) self.base._report_icase_hint(pkg_spec) self._report_alternatives(pkg_spec) errs.append(pkg_spec) return errs updateinfo.py 0000644 00000045072 15025176534 0007274 0 ustar 00 # updateinfo.py # UpdateInfo CLI command. # # Copyright (C) 2014-2016 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, # modify, copy, or redistribute it subject to the terms and conditions of # the GNU General Public License v.2, or (at your option) any later version. # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY expressed or implied, including the implied warranties of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. You should have received a copy of the # GNU General Public License along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. Any Red Hat trademarks that are incorporated in the # source code or documentation are not subject to the GNU General Public # License and may only be used or replicated with the express permission of # Red Hat, Inc. # """UpdateInfo CLI command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import collections import fnmatch import hawkey from dnf.cli import commands from dnf.cli.option_parser import OptionParser from dnf.i18n import _, exact_width from dnf.pycomp import unicode def _maxlen(iterable): """Return maximum length of items in a non-empty iterable.""" return max(exact_width(item) for item in iterable) class UpdateInfoCommand(commands.Command): """Implementation of the UpdateInfo command.""" TYPE2LABEL = {hawkey.ADVISORY_BUGFIX: _('bugfix'), hawkey.ADVISORY_ENHANCEMENT: _('enhancement'), hawkey.ADVISORY_SECURITY: _('security'), hawkey.ADVISORY_UNKNOWN: _('unknown'), hawkey.ADVISORY_NEWPACKAGE: _('newpackage')} SECURITY2LABEL = {'Critical': _('Critical/Sec.'), 'Important': _('Important/Sec.'), 'Moderate': _('Moderate/Sec.'), 'Low': _('Low/Sec.')} direct_commands = {'list-updateinfo' : 'list', 'list-security' : 'list', 'list-sec' : 'list', 'info-updateinfo' : 'info', 'info-security' : 'info', 'info-sec' : 'info', 'summary-updateinfo' : 'summary'} aliases = ['updateinfo', 'upif'] + list(direct_commands.keys()) summary = _('display advisories about packages') availability_default = 'available' availabilities = ['installed', 'updates', 'all', availability_default] def __init__(self, cli): """Initialize the command.""" super(UpdateInfoCommand, self).__init__(cli) self._installed_query = None @staticmethod def set_argparser(parser): availability = parser.add_mutually_exclusive_group() availability.add_argument( "--available", dest='_availability', const='available', action='store_const', help=_("advisories about newer versions of installed packages (default)")) availability.add_argument( "--installed", dest='_availability', const='installed', action='store_const', help=_("advisories about equal and older versions of installed packages")) availability.add_argument( "--updates", dest='_availability', const='updates', action='store_const', help=_("advisories about newer versions of those installed packages " "for which a newer version is available")) availability.add_argument( "--all", dest='_availability', const='all', action='store_const', help=_("advisories about any versions of installed packages")) cmds = ['summary', 'list', 'info'] output_format = parser.add_mutually_exclusive_group() output_format.add_argument("--summary", dest='_spec_action', const='summary', action='store_const', help=_('show summary of advisories (default)')) output_format.add_argument("--list", dest='_spec_action', const='list', action='store_const', help=_('show list of advisories')) output_format.add_argument("--info", dest='_spec_action', const='info', action='store_const', help=_('show info of advisories')) parser.add_argument("--with-cve", dest='with_cve', default=False, action='store_true', help=_('show only advisories with CVE reference')) parser.add_argument("--with-bz", dest='with_bz', default=False, action='store_true', help=_('show only advisories with bugzilla reference')) parser.add_argument('spec', nargs='*', metavar='SPEC', choices=cmds, default=cmds[0], action=OptionParser.PkgNarrowCallback, help=_("Package specification")) def configure(self): """Do any command-specific configuration based on command arguments.""" self.cli.demands.available_repos = True self.cli.demands.sack_activation = True if self.opts.command in self.direct_commands: # we were called with direct command self.opts.spec_action = self.direct_commands[self.opts.command] else: if self.opts._spec_action: self.opts.spec_action = self.opts._spec_action if self.opts._availability: self.opts.availability = self.opts._availability else: # yum compatibility - search for all|available|installed|updates in spec[0] if not self.opts.spec or self.opts.spec[0] not in self.availabilities: self.opts.availability = self.availability_default else: self.opts.availability = self.opts.spec.pop(0) # filtering by advisory types (security/bugfix/enhancement/newpackage) self.opts._advisory_types = set() if self.opts.bugfix: self.opts._advisory_types.add(hawkey.ADVISORY_BUGFIX) if self.opts.enhancement: self.opts._advisory_types.add(hawkey.ADVISORY_ENHANCEMENT) if self.opts.newpackage: self.opts._advisory_types.add(hawkey.ADVISORY_NEWPACKAGE) if self.opts.security: self.opts._advisory_types.add(hawkey.ADVISORY_SECURITY) # yum compatibility - yum accepts types also as positional arguments if self.opts.spec: spec = self.opts.spec.pop(0) if spec == 'bugfix': self.opts._advisory_types.add(hawkey.ADVISORY_BUGFIX) elif spec == 'enhancement': self.opts._advisory_types.add(hawkey.ADVISORY_ENHANCEMENT) elif spec in ('security', 'sec'): self.opts._advisory_types.add(hawkey.ADVISORY_SECURITY) elif spec == 'newpackage': self.opts._advisory_types.add(hawkey.ADVISORY_NEWPACKAGE) elif spec in ('bugzillas', 'bzs'): self.opts.with_bz = True elif spec == 'cves': self.opts.with_cve = True else: self.opts.spec.insert(0, spec) if self.opts.advisory: self.opts.spec.extend(self.opts.advisory) def run(self): """Execute the command with arguments.""" if self.opts.availability == 'installed': apkg_adv_insts = self.installed_apkg_adv_insts(self.opts.spec) description = _('installed') elif self.opts.availability == 'updates': apkg_adv_insts = self.updating_apkg_adv_insts(self.opts.spec) description = _('updates') elif self.opts.availability == 'all': apkg_adv_insts = self.all_apkg_adv_insts(self.opts.spec) description = _('all') else: apkg_adv_insts = self.available_apkg_adv_insts(self.opts.spec) description = _('available') if self.opts.spec_action == 'list': self.display_list(apkg_adv_insts) elif self.opts.spec_action == 'info': self.display_info(apkg_adv_insts) else: self.display_summary(apkg_adv_insts, description) def _newer_equal_installed(self, apackage): if self._installed_query is None: self._installed_query = self.base.sack.query().installed().apply() q = self._installed_query.filter(name=apackage.name, evr__gte=apackage.evr) return len(q) > 0 def _advisory_matcher(self, advisory): if not self.opts._advisory_types \ and not self.opts.spec \ and not self.opts.severity \ and not self.opts.bugzilla \ and not self.opts.cves \ and not self.opts.with_cve \ and not self.opts.with_bz: return True if advisory.type in self.opts._advisory_types: return True if any(fnmatch.fnmatchcase(advisory.id, pat) for pat in self.opts.spec): return True if self.opts.severity and advisory.severity in self.opts.severity: return True if self.opts.bugzilla and any([advisory.match_bug(bug) for bug in self.opts.bugzilla]): return True if self.opts.cves and any([advisory.match_cve(cve) for cve in self.opts.cves]): return True if self.opts.with_cve: if any([ref.type == hawkey.REFERENCE_CVE for ref in advisory.references]): return True if self.opts.with_bz: if any([ref.type == hawkey.REFERENCE_BUGZILLA for ref in advisory.references]): return True return False def _apackage_advisory_installed(self, pkgs_query, cmptype, specs): """Return (adv. package, advisory, installed) triplets.""" for apackage in pkgs_query.get_advisory_pkgs(cmptype): advisory = apackage.get_advisory(self.base.sack) advisory_match = self._advisory_matcher(advisory) apackage_match = any(fnmatch.fnmatchcase(apackage.name, pat) for pat in self.opts.spec) if advisory_match or apackage_match: installed = self._newer_equal_installed(apackage) yield apackage, advisory, installed def running_kernel_pkgs(self): """Return query containing packages of currently running kernel""" sack = self.base.sack q = sack.query().filterm(empty=True) kernel = sack.get_running_kernel() if kernel: q = q.union(sack.query().filterm(sourcerpm=kernel.sourcerpm)) return q def available_apkg_adv_insts(self, specs): """Return available (adv. package, adv., inst.) triplets""" # check advisories for the latest installed packages q = self.base.sack.query().installed().latest(1) # plus packages of the running kernel q = q.union(self.running_kernel_pkgs().installed()) return self._apackage_advisory_installed(q, hawkey.GT, specs) def installed_apkg_adv_insts(self, specs): """Return installed (adv. package, adv., inst.) triplets""" return self._apackage_advisory_installed( self.base.sack.query().installed(), hawkey.LT | hawkey.EQ, specs) def updating_apkg_adv_insts(self, specs): """Return updating (adv. package, adv., inst.) triplets""" return self._apackage_advisory_installed( self.base.sack.query().filterm(upgradable=True), hawkey.GT, specs) def all_apkg_adv_insts(self, specs): """Return installed (adv. package, adv., inst.) triplets""" return self._apackage_advisory_installed( self.base.sack.query().installed(), hawkey.LT | hawkey.EQ | hawkey.GT, specs) def _summary(self, apkg_adv_insts): """Make the summary of advisories.""" # Remove duplicate advisory IDs. We assume that the ID is unique within # a repository and two advisories with the same IDs in different # repositories must have the same type. id2type = {} for (apkg, advisory, installed) in apkg_adv_insts: id2type[advisory.id] = advisory.type if advisory.type == hawkey.ADVISORY_SECURITY: id2type[(advisory.id, advisory.severity)] = (advisory.type, advisory.severity) return collections.Counter(id2type.values()) def display_summary(self, apkg_adv_insts, description): """Display the summary of advisories.""" typ2cnt = self._summary(apkg_adv_insts) if typ2cnt: print(_('Updates Information Summary: ') + description) # Convert types to strings and order the entries. label_counts = [ (0, _('New Package notice(s)'), typ2cnt[hawkey.ADVISORY_NEWPACKAGE]), (0, _('Security notice(s)'), typ2cnt[hawkey.ADVISORY_SECURITY]), (1, _('Critical Security notice(s)'), typ2cnt[(hawkey.ADVISORY_SECURITY, 'Critical')]), (1, _('Important Security notice(s)'), typ2cnt[(hawkey.ADVISORY_SECURITY, 'Important')]), (1, _('Moderate Security notice(s)'), typ2cnt[(hawkey.ADVISORY_SECURITY, 'Moderate')]), (1, _('Low Security notice(s)'), typ2cnt[(hawkey.ADVISORY_SECURITY, 'Low')]), (1, _('Unknown Security notice(s)'), typ2cnt[(hawkey.ADVISORY_SECURITY, None)]), (0, _('Bugfix notice(s)'), typ2cnt[hawkey.ADVISORY_BUGFIX]), (0, _('Enhancement notice(s)'), typ2cnt[hawkey.ADVISORY_ENHANCEMENT]), (0, _('other notice(s)'), typ2cnt[hawkey.ADVISORY_UNKNOWN])] width = _maxlen(unicode(v[2]) for v in label_counts if v[2]) for indent, label, count in label_counts: if not count: continue print(' %*s %s' % (width + 4 * indent, unicode(count), label)) if self.base.conf.autocheck_running_kernel: self.cli._check_running_kernel() def display_list(self, apkg_adv_insts): """Display the list of advisories.""" def inst2mark(inst): if not self.opts.availability == 'all': return '' elif inst: return 'i ' else: return ' ' def type2label(typ, sev): if typ == hawkey.ADVISORY_SECURITY: return self.SECURITY2LABEL.get(sev, _('Unknown/Sec.')) else: return self.TYPE2LABEL.get(typ, _('unknown')) nevra_inst_dict = dict() for apkg, advisory, installed in apkg_adv_insts: nevra = '%s-%s.%s' % (apkg.name, apkg.evr, apkg.arch) if self.opts.with_cve or self.opts.with_bz: for ref in advisory.references: if ref.type == hawkey.REFERENCE_BUGZILLA and not self.opts.with_bz: continue elif ref.type == hawkey.REFERENCE_CVE and not self.opts.with_cve: continue nevra_inst_dict.setdefault((nevra, installed, advisory.updated), dict())[ref.id] = ( advisory.type, advisory.severity) else: nevra_inst_dict.setdefault((nevra, installed, advisory.updated), dict())[advisory.id] = ( advisory.type, advisory.severity) advlist = [] # convert types to labels, find max len of advisory IDs and types idw = tlw = nw = 0 for (nevra, inst, aupdated), id2type in sorted(nevra_inst_dict.items(), key=lambda x: x[0]): nw = max(nw, len(nevra)) for aid, atypesev in id2type.items(): idw = max(idw, len(aid)) label = type2label(*atypesev) tlw = max(tlw, len(label)) advlist.append((inst2mark(inst), aid, label, nevra, aupdated)) for (inst, aid, label, nevra, aupdated) in advlist: if self.base.conf.verbose: print('%s%-*s %-*s %-*s %s' % (inst, idw, aid, tlw, label, nw, nevra, aupdated)) else: print('%s%-*s %-*s %s' % (inst, idw, aid, tlw, label, nevra)) def display_info(self, apkg_adv_insts): """Display the details about available advisories.""" arches = self.base.sack.list_arches() verbose = self.base.conf.verbose labels = (_('Update ID'), _('Type'), _('Updated'), _('Bugs'), _('CVEs'), _('Description'), _('Severity'), _('Rights'), _('Files'), _('Installed')) def advisory2info(advisory, installed): attributes = [ [advisory.id], [self.TYPE2LABEL.get(advisory.type, _('unknown'))], [unicode(advisory.updated)], [], [], (advisory.description or '').splitlines(), [advisory.severity], (advisory.rights or '').splitlines(), sorted(set(pkg.filename for pkg in advisory.packages if pkg.arch in arches)), None] for ref in advisory.references: if ref.type == hawkey.REFERENCE_BUGZILLA: attributes[3].append('{} - {}'.format(ref.id, ref.title or '')) elif ref.type == hawkey.REFERENCE_CVE: attributes[4].append(ref.id) attributes[3].sort() attributes[4].sort() if not verbose: attributes[7] = None attributes[8] = None if self.opts.availability == 'all': attributes[9] = [_('true') if installed else _('false')] width = _maxlen(labels) lines = [] lines.append('=' * 79) lines.append(' ' + advisory.title) lines.append('=' * 79) for label, atr_lines in zip(labels, attributes): if atr_lines in (None, [None]): continue for i, line in enumerate(atr_lines): key = label if i == 0 else '' key_padding = width - exact_width(key) lines.append('%*s%s: %s' % (key_padding, "", key, line)) return '\n'.join(lines) advisories = set() for apkg, advisory, installed in apkg_adv_insts: advisories.add(advisory2info(advisory, installed)) print("\n\n".join(sorted(advisories, key=lambda x: x.lower()))) history.py 0000644 00000043045 15025176534 0006635 0 ustar 00 # Copyright 2006 Duke University # Copyright (C) 2012-2016 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals import libdnf import hawkey from dnf.i18n import _, ucd from dnf.cli import commands from dnf.transaction_sr import TransactionReplay, serialize_transaction import dnf.cli import dnf.exceptions import dnf.transaction import dnf.util import json import logging import os logger = logging.getLogger('dnf') class HistoryCommand(commands.Command): """A class containing methods needed by the cli to execute the history command. """ aliases = ('history', 'hist') summary = _('display, or use, the transaction history') _CMDS = ['list', 'info', 'redo', 'replay', 'rollback', 'store', 'undo', 'userinstalled'] def __init__(self, *args, **kw): super(HistoryCommand, self).__init__(*args, **kw) self._require_one_transaction_id = False @staticmethod def set_argparser(parser): parser.add_argument('transactions_action', nargs='?', metavar="COMMAND", help="Available commands: {} (default), {}".format( HistoryCommand._CMDS[0], ", ".join(HistoryCommand._CMDS[1:]))) parser.add_argument('--reverse', action='store_true', help="display history list output reversed") parser.add_argument("-o", "--output", default=None, help=_("For the store command, file path to store the transaction to")) parser.add_argument("--ignore-installed", action="store_true", help=_("For the replay command, don't check for installed packages matching " "those in transaction")) parser.add_argument("--ignore-extras", action="store_true", help=_("For the replay command, don't check for extra packages pulled " "into the transaction")) parser.add_argument("--skip-unavailable", action="store_true", help=_("For the replay command, skip packages that are not available or have " "missing dependencies")) parser.add_argument('transactions', nargs='*', metavar="TRANSACTION", help="For commands working with history transactions, " "Transaction ID (<number>, 'last' or 'last-<number>' " "for one transaction, <transaction-id>..<transaction-id> " "for a range)") parser.add_argument('transaction_filename', nargs='?', metavar="TRANSACTION_FILE", help="For the replay command, path to the stored " "transaction file to replay") def configure(self): if not self.opts.transactions_action: # no positional argument given self.opts.transactions_action = self._CMDS[0] elif self.opts.transactions_action not in self._CMDS: # first positional argument is not a command self.opts.transactions.insert(0, self.opts.transactions_action) self.opts.transactions_action = self._CMDS[0] self._require_one_transaction_id_msg = _("Found more than one transaction ID.\n" "'{}' requires one transaction ID or package name." ).format(self.opts.transactions_action) demands = self.cli.demands if self.opts.transactions_action == 'replay': if not self.opts.transactions: raise dnf.cli.CliError(_('No transaction file name given.')) if len(self.opts.transactions) > 1: raise dnf.cli.CliError(_('More than one argument given as transaction file name.')) # in case of replay, copy over the file name to it's appropriate variable # (the arg parser can't distinguish here) self.opts.transaction_filename = os.path.abspath(self.opts.transactions[0]) self.opts.transactions = [] demands.available_repos = True demands.resolving = True demands.root_user = True # Override configuration options that affect how the transaction is resolved self.base.conf.clean_requirements_on_remove = False self.base.conf.install_weak_deps = False dnf.cli.commands._checkGPGKey(self.base, self.cli) elif self.opts.transactions_action == 'store': self._require_one_transaction_id = True if not self.opts.transactions: raise dnf.cli.CliError(_('No transaction ID or package name given.')) elif self.opts.transactions_action in ['redo', 'undo', 'rollback']: demands.available_repos = True demands.resolving = True demands.root_user = True self._require_one_transaction_id = True if not self.opts.transactions: msg = _('No transaction ID or package name given.') logger.critical(msg) raise dnf.cli.CliError(msg) elif len(self.opts.transactions) > 1: logger.critical(self._require_one_transaction_id_msg) raise dnf.cli.CliError(self._require_one_transaction_id_msg) demands.available_repos = True dnf.cli.commands._checkGPGKey(self.base, self.cli) else: demands.fresh_metadata = False demands.sack_activation = True if self.base.history.path != ":memory:" and not os.access(self.base.history.path, os.R_OK): msg = _("You don't have access to the history DB: %s" % self.base.history.path) logger.critical(msg) raise dnf.cli.CliError(msg) def get_error_output(self, error): """Get suggestions for resolving the given error.""" if isinstance(error, dnf.exceptions.TransactionCheckError): if self.opts.transactions_action == 'undo': id_, = self.opts.transactions return (_('Cannot undo transaction %s, doing so would result ' 'in an inconsistent package database.') % id_,) elif self.opts.transactions_action == 'rollback': id_, = (self.opts.transactions if self.opts.transactions[0] != 'force' else self.opts.transactions[1:]) return (_('Cannot rollback transaction %s, doing so would ' 'result in an inconsistent package database.') % id_,) return dnf.cli.commands.Command.get_error_output(self, error) def _hcmd_redo(self, extcmds): old = self._history_get_transaction(extcmds) data = serialize_transaction(old) self.replay = TransactionReplay( self.base, data=data, ignore_installed=True, ignore_extras=True, skip_unavailable=self.opts.skip_unavailable ) self.replay.run() def _history_get_transactions(self, extcmds): if not extcmds: raise dnf.cli.CliError(_('No transaction ID given')) old = self.base.history.old(extcmds) if not old: raise dnf.cli.CliError(_('Transaction ID "{0}" not found.').format(extcmds[0])) return old def _history_get_transaction(self, extcmds): old = self._history_get_transactions(extcmds) if len(old) > 1: raise dnf.cli.CliError(_('Found more than one transaction ID!')) return old[0] def _hcmd_undo(self, extcmds): old = self._history_get_transaction(extcmds) self._revert_transaction(old) def _hcmd_rollback(self, extcmds): old = self._history_get_transaction(extcmds) last = self.base.history.last() merged_trans = None if old.tid != last.tid: # history.old([]) returns all transactions and we don't want that # so skip merging the transactions when trying to rollback to the last transaction # which is the current system state and rollback is not applicable for trans in self.base.history.old(list(range(old.tid + 1, last.tid + 1))): if trans.altered_lt_rpmdb: logger.warning(_('Transaction history is incomplete, before %u.'), trans.tid) elif trans.altered_gt_rpmdb: logger.warning(_('Transaction history is incomplete, after %u.'), trans.tid) if merged_trans is None: merged_trans = dnf.db.history.MergedTransactionWrapper(trans) else: merged_trans.merge(trans) self._revert_transaction(merged_trans) def _revert_transaction(self, trans): action_map = { "Install": "Removed", "Removed": "Install", "Upgrade": "Downgraded", "Upgraded": "Downgrade", "Downgrade": "Upgraded", "Downgraded": "Upgrade", "Reinstalled": "Reinstall", "Reinstall": "Reinstalled", "Obsoleted": "Install", "Obsolete": "Obsoleted", "Reason Change": "Reason Change", } data = serialize_transaction(trans) # revert actions in the serialized transaction data to perform rollback/undo for content_type in ("rpms", "groups", "environments"): for ti in data.get(content_type, []): ti["action"] = action_map[ti["action"]] if ti["action"] == "Install" and ti.get("reason", None) == "clean": ti["reason"] = "dependency" if ti["action"] == "Reason Change" and "nevra" in ti: subj = hawkey.Subject(ti["nevra"]) nevra = subj.get_nevra_possibilities(forms=[hawkey.FORM_NEVRA])[0] reason = self.output.history.swdb.resolveRPMTransactionItemReason( nevra.name, nevra.arch, trans.tids()[0] - 1 ) ti["reason"] = libdnf.transaction.TransactionItemReasonToString(reason) if ti.get("repo_id") == hawkey.SYSTEM_REPO_NAME: # erase repo_id, because it's not possible to perform forward actions from the @System repo ti["repo_id"] = None self.replay = TransactionReplay( self.base, data=data, ignore_installed=True, ignore_extras=True, skip_unavailable=self.opts.skip_unavailable ) self.replay.run() def _hcmd_userinstalled(self): """Execute history userinstalled command.""" pkgs = tuple(self.base.iter_userinstalled()) n_listed = self.output.listPkgs(pkgs, 'Packages installed by user', 'nevra') if n_listed == 0: raise dnf.cli.CliError(_('No packages to list')) def _args2transaction_ids(self): """Convert commandline arguments to transaction ids""" def str2transaction_id(s): if s == 'last': s = '0' elif s.startswith('last-'): s = s[4:] transaction_id = int(s) if transaction_id <= 0: transaction_id += self.output.history.last().tid return transaction_id tids = set() merged_tids = set() for t in self.opts.transactions: if '..' in t: try: begin_transaction_id, end_transaction_id = t.split('..', 2) except ValueError: logger.critical( _("Invalid transaction ID range definition '{}'.\n" "Use '<transaction-id>..<transaction-id>'." ).format(t)) raise dnf.cli.CliError cant_convert_msg = _("Can't convert '{}' to transaction ID.\n" "Use '<number>', 'last', 'last-<number>'.") try: begin_transaction_id = str2transaction_id(begin_transaction_id) except ValueError: logger.critical(_(cant_convert_msg).format(begin_transaction_id)) raise dnf.cli.CliError try: end_transaction_id = str2transaction_id(end_transaction_id) except ValueError: logger.critical(_(cant_convert_msg).format(end_transaction_id)) raise dnf.cli.CliError if self._require_one_transaction_id and begin_transaction_id != end_transaction_id: logger.critical(self._require_one_transaction_id_msg) raise dnf.cli.CliError if begin_transaction_id > end_transaction_id: begin_transaction_id, end_transaction_id = \ end_transaction_id, begin_transaction_id merged_tids.add((begin_transaction_id, end_transaction_id)) tids.update(range(begin_transaction_id, end_transaction_id + 1)) else: try: tids.add(str2transaction_id(t)) except ValueError: # not a transaction id, assume it's package name transact_ids_from_pkgname = self.output.history.search([t]) if transact_ids_from_pkgname: tids.update(transact_ids_from_pkgname) else: msg = _("No transaction which manipulates package '{}' was found." ).format(t) if self._require_one_transaction_id: logger.critical(msg) raise dnf.cli.CliError else: logger.info(msg) return sorted(tids, reverse=True), merged_tids def run(self): vcmd = self.opts.transactions_action if vcmd == 'replay': self.replay = TransactionReplay( self.base, filename=self.opts.transaction_filename, ignore_installed = self.opts.ignore_installed, ignore_extras = self.opts.ignore_extras, skip_unavailable = self.opts.skip_unavailable ) self.replay.run() else: tids, merged_tids = self._args2transaction_ids() if vcmd == 'list' and (tids or not self.opts.transactions): self.output.historyListCmd(tids, reverse=self.opts.reverse) elif vcmd == 'info' and (tids or not self.opts.transactions): self.output.historyInfoCmd(tids, self.opts.transactions, merged_tids) elif vcmd == 'undo': self._hcmd_undo(tids) elif vcmd == 'redo': self._hcmd_redo(tids) elif vcmd == 'rollback': self._hcmd_rollback(tids) elif vcmd == 'userinstalled': self._hcmd_userinstalled() elif vcmd == 'store': tid = self._history_get_transaction(tids) data = serialize_transaction(tid) try: filename = self.opts.output if self.opts.output is not None else "transaction.json" # it is absolutely possible for both assumeyes and assumeno to be True, go figure if (self.base.conf.assumeno or not self.base.conf.assumeyes) and os.path.isfile(filename): msg = _("{} exists, overwrite?").format(filename) if self.base.conf.assumeno or not self.base.output.userconfirm( msg='\n{} [y/N]: '.format(msg), defaultyes_msg='\n{} [Y/n]: '.format(msg)): print(_("Not overwriting {}, exiting.").format(filename)) return with open(filename, "w") as f: json.dump(data, f, indent=4, sort_keys=True) f.write("\n") print(_("Transaction saved to {}.").format(filename)) except OSError as e: raise dnf.cli.CliError(_('Error storing transaction: {}').format(str(e))) def run_resolved(self): if self.opts.transactions_action not in ("replay", "redo", "rollback", "undo"): return self.replay.post_transaction() def run_transaction(self): if self.opts.transactions_action not in ("replay", "redo", "rollback", "undo"): return warnings = self.replay.get_warnings() if warnings: logger.log( dnf.logging.WARNING, _("Warning, the following problems occurred while running a transaction:") ) for w in warnings: logger.log(dnf.logging.WARNING, " " + w)
| ver. 1.4 |
Github
|
.
| PHP 8.2.28 | Generation time: 0.05 |
proxy
|
phpinfo
|
Settings