5 * Mass import-export and batch import functionality for Gettext .po files.
8 use Drupal\Core\Language\LanguageInterface;
9 use Drupal\file\FileInterface;
10 use Drupal\locale\Gettext;
11 use Drupal\locale\Locale;
14 * Prepare a batch to import all translations.
16 * @param array $options
17 * An array with options that can have the following elements:
18 * - 'langcode': The language code. Optional, defaults to NULL, which means
19 * that the language will be detected from the name of the files.
20 * - 'overwrite_options': Overwrite options array as defined in
21 * Drupal\locale\PoDatabaseWriter. Optional, defaults to an empty array.
22 * - 'customized': Flag indicating whether the strings imported from $file
23 * are customized translations or come from a community source. Use
24 * LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. Optional, defaults to
25 * LOCALE_NOT_CUSTOMIZED.
26 * - 'finish_feedback': Whether or not to give feedback to the user when the
27 * batch is finished. Optional, defaults to TRUE.
29 * (optional) Import all available files, even if they were imported before.
32 * The batch structure, or FALSE if no files are used to build the batch.
35 * Integrate with update status to identify projects needed and integrate
36 * l10n_update functionality to feed in translation files alike.
37 * See https://www.drupal.org/node/1191488.
39 function locale_translate_batch_import_files(array $options, $force = FALSE) {
41 'overwrite_options' => [],
42 'customized' => LOCALE_NOT_CUSTOMIZED,
43 'finish_feedback' => TRUE,
46 if (!empty($options['langcode'])) {
47 $langcodes = [$options['langcode']];
50 // If langcode was not provided, make sure to only import files for the
51 // languages we have added.
52 $langcodes = array_keys(\Drupal::languageManager()->getLanguages());
55 $files = locale_translate_get_interface_translation_files([], $langcodes);
58 $result = db_select('locale_file', 'lf')
59 ->fields('lf', ['langcode', 'uri', 'timestamp'])
60 ->condition('langcode', $langcodes)
62 ->fetchAllAssoc('uri');
63 foreach ($result as $uri => $info) {
64 if (isset($files[$uri]) && filemtime($uri) <= $info->timestamp) {
65 // The file is already imported and not changed since the last import.
66 // Remove it from file list and don't import it again.
71 return locale_translate_batch_build($files, $options);
75 * Get interface translation files present in the translations directory.
77 * @param array $projects
78 * (optional) Project names from which to get the translation files and
79 * history. Defaults to all projects.
80 * @param array $langcodes
81 * (optional) Language codes from which to get the translation files and
82 * history. Defaults to all languages.
85 * An array of interface translation files keyed by their URI.
87 function locale_translate_get_interface_translation_files(array $projects = [], array $langcodes = []) {
88 module_load_include('compare.inc', 'locale');
90 $projects = $projects ? $projects : array_keys(locale_translation_get_projects());
91 $langcodes = $langcodes ? $langcodes : array_keys(locale_translatable_language_list());
93 // Scan the translations directory for files matching a name pattern
94 // containing a project name and language code: {project}.{langcode}.po or
95 // {project}-{version}.{langcode}.po.
96 // Only files of known projects and languages will be returned.
97 $directory = \Drupal::config('locale.settings')->get('translation.path');
98 $result = file_scan_directory($directory, '![a-z_]+(\-[0-9a-z\.\-\+]+|)\.[^\./]+\.po$!', ['recurse' => FALSE]);
100 foreach ($result as $file) {
101 // Update the file object with project name and version from the file name.
102 $file = locale_translate_file_attach_properties($file);
103 if (in_array($file->project, $projects)) {
104 if (in_array($file->langcode, $langcodes)) {
105 $files[$file->uri] = $file;
114 * Build a locale batch from an array of files.
116 * @param array $files
117 * Array of file objects to import.
118 * @param array $options
119 * An array with options that can have the following elements:
120 * - 'langcode': The language code. Optional, defaults to NULL, which means
121 * that the language will be detected from the name of the files.
122 * - 'overwrite_options': Overwrite options array as defined in
123 * Drupal\locale\PoDatabaseWriter. Optional, defaults to an empty array.
124 * - 'customized': Flag indicating whether the strings imported from $file
125 * are customized translations or come from a community source. Use
126 * LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. Optional, defaults to
127 * LOCALE_NOT_CUSTOMIZED.
128 * - 'finish_feedback': Whether or not to give feedback to the user when the
129 * batch is finished. Optional, defaults to TRUE.
132 * A batch structure or FALSE if $files was empty.
134 function locale_translate_batch_build(array $files, array $options) {
136 'overwrite_options' => [],
137 'customized' => LOCALE_NOT_CUSTOMIZED,
138 'finish_feedback' => TRUE,
142 foreach ($files as $file) {
143 // We call locale_translate_batch_import for every batch operation.
144 $operations[] = ['locale_translate_batch_import', [$file, $options]];
146 // Save the translation status of all files.
147 $operations[] = ['locale_translate_batch_import_save', []];
149 // Add a final step to refresh JavaScript and configuration strings.
150 $operations[] = ['locale_translate_batch_refresh', []];
153 'operations' => $operations,
154 'title' => t('Importing interface translations'),
155 'progress_message' => '',
156 'error_message' => t('Error importing interface translations'),
157 'file' => drupal_get_path('module', 'locale') . '/locale.bulk.inc',
159 if ($options['finish_feedback']) {
160 $batch['finished'] = 'locale_translate_batch_finished';
168 * Implements callback_batch_operation().
170 * Perform interface translation import.
172 * @param object $file
173 * A file object of the gettext file to be imported. The file object must
174 * contain a language parameter (other than
175 * LanguageInterface::LANGCODE_NOT_SPECIFIED). This is used as the language of
177 * @param array $options
178 * An array with options that can have the following elements:
179 * - 'langcode': The language code.
180 * - 'overwrite_options': Overwrite options array as defined in
181 * Drupal\locale\PoDatabaseWriter. Optional, defaults to an empty array.
182 * - 'customized': Flag indicating whether the strings imported from $file
183 * are customized translations or come from a community source. Use
184 * LOCALE_CUSTOMIZED or LOCALE_NOT_CUSTOMIZED. Optional, defaults to
185 * LOCALE_NOT_CUSTOMIZED.
186 * - 'message': Alternative message to display during import. Note, this must
188 * @param array|\ArrayAccess $context
189 * Contains a list of files imported.
191 function locale_translate_batch_import($file, array $options, &$context) {
192 // Merge the default values in the $options array.
194 'overwrite_options' => [],
195 'customized' => LOCALE_NOT_CUSTOMIZED,
198 if (isset($file->langcode) && $file->langcode != LanguageInterface::LANGCODE_NOT_SPECIFIED) {
201 if (empty($context['sandbox'])) {
202 $context['sandbox']['parse_state'] = [
203 'filesize' => filesize(drupal_realpath($file->uri)),
208 // Update the seek and the number of items in the $options array().
209 $options['seek'] = $context['sandbox']['parse_state']['seek'];
210 $options['items'] = $context['sandbox']['parse_state']['chunk_size'];
211 $report = Gettext::fileToDatabase($file, $options);
212 // If not yet finished with reading, mark progress based on size and
214 if ($report['seek'] < filesize($file->uri)) {
216 $context['sandbox']['parse_state']['seek'] = $report['seek'];
217 // Maximize the progress bar at 95% before completion, the batch API
218 // could trigger the end of the operation before file reading is done,
219 // because of floating point inaccuracies. See
220 // https://www.drupal.org/node/1089472.
221 $context['finished'] = min(0.95, $report['seek'] / filesize($file->uri));
222 if (isset($options['message'])) {
223 $context['message'] = t('@message (@percent%).', ['@message' => $options['message'], '@percent' => (int) ($context['finished'] * 100)]);
226 $context['message'] = t('Importing translation file: %filename (@percent%).', ['%filename' => $file->filename, '@percent' => (int) ($context['finished'] * 100)]);
230 // We are finished here.
231 $context['finished'] = 1;
233 // Store the file data for processing by the next batch operation.
234 $file->timestamp = filemtime($file->uri);
235 $context['results']['files'][$file->uri] = $file;
236 $context['results']['languages'][$file->uri] = $file->langcode;
239 // Add the reported values to the statistics for this file.
240 // Each import iteration reports statistics in an array. The results of
241 // each iteration are added and merged here and stored per file.
242 if (!isset($context['results']['stats']) || !isset($context['results']['stats'][$file->uri])) {
243 $context['results']['stats'][$file->uri] = [];
245 foreach ($report as $key => $value) {
246 if (is_numeric($report[$key])) {
247 if (!isset($context['results']['stats'][$file->uri][$key])) {
248 $context['results']['stats'][$file->uri][$key] = 0;
250 $context['results']['stats'][$file->uri][$key] += $report[$key];
252 elseif (is_array($value)) {
253 $context['results']['stats'][$file->uri] += [$key => []];
254 $context['results']['stats'][$file->uri][$key] = array_merge($context['results']['stats'][$file->uri][$key], $value);
258 catch (Exception $exception) {
259 // Import failed. Store the data of the failing file.
260 $context['results']['failed_files'][] = $file;
261 \Drupal::logger('locale')->notice('Unable to import translations file: @file', ['@file' => $file->uri]);
267 * Implements callback_batch_operation().
269 * Save data of imported files.
271 * @param array|\ArrayAccess $context
272 * Contains a list of imported files.
274 function locale_translate_batch_import_save($context) {
275 if (isset($context['results']['files'])) {
276 foreach ($context['results']['files'] as $file) {
277 // Update the file history if both project and version are known. This
278 // table is used by the automated translation update function which tracks
279 // translation status of module and themes in the system. Other
280 // translation files are not tracked and are therefore not stored in this
282 if ($file->project && $file->version) {
283 $file->last_checked = REQUEST_TIME;
284 locale_translation_update_file_history($file);
287 $context['message'] = t('Translations imported.');
292 * Implements callback_batch_operation().
294 * Refreshes translations after importing strings.
296 * @param array|\ArrayAccess $context
297 * Contains a list of strings updated and information about the progress.
299 function locale_translate_batch_refresh(&$context) {
300 if (!isset($context['sandbox']['refresh'])) {
301 $strings = $langcodes = [];
302 if (isset($context['results']['stats'])) {
303 // Get list of unique string identifiers and language codes updated.
304 $langcodes = array_unique(array_values($context['results']['languages']));
305 foreach ($context['results']['stats'] as $report) {
306 $strings = array_merge($strings, $report['strings']);
310 // Initialize multi-step string refresh.
311 $context['message'] = t('Updating translations for JavaScript and default configuration.');
312 $context['sandbox']['refresh']['strings'] = array_unique($strings);
313 $context['sandbox']['refresh']['languages'] = $langcodes;
314 $context['sandbox']['refresh']['names'] = [];
315 $context['results']['stats']['config'] = 0;
316 $context['sandbox']['refresh']['count'] = count($strings);
318 // We will update strings on later steps.
319 $context['finished'] = 0;
322 $context['finished'] = 1;
325 elseif ($name = array_shift($context['sandbox']['refresh']['names'])) {
326 // Refresh all languages for one object at a time.
327 $count = Locale::config()->updateConfigTranslations([$name], $context['sandbox']['refresh']['languages']);
328 $context['results']['stats']['config'] += $count;
329 // Inherit finished information from the "parent" string lookup step so
330 // visual display of status will make sense.
331 $context['finished'] = $context['sandbox']['refresh']['names_finished'];
332 $context['message'] = t('Updating default configuration (@percent%).', ['@percent' => (int) ($context['finished'] * 100)]);
334 elseif (!empty($context['sandbox']['refresh']['strings'])) {
335 // Not perfect but will give some indication of progress.
336 $context['finished'] = 1 - count($context['sandbox']['refresh']['strings']) / $context['sandbox']['refresh']['count'];
337 // Pending strings, refresh 100 at a time, get next pack.
338 $next = array_slice($context['sandbox']['refresh']['strings'], 0, 100);
339 array_splice($context['sandbox']['refresh']['strings'], 0, count($next));
340 // Clear cache and force refresh of JavaScript translations.
341 _locale_refresh_translations($context['sandbox']['refresh']['languages'], $next);
342 // Check whether we need to refresh configuration objects.
343 if ($names = Locale::config()->getStringNames($next)) {
344 $context['sandbox']['refresh']['names_finished'] = $context['finished'];
345 $context['sandbox']['refresh']['names'] = $names;
349 $context['message'] = t('Updated default configuration.');
350 $context['finished'] = 1;
355 * Implements callback_batch_finished().
357 * Finished callback of system page locale import batch.
359 * @param bool $success
360 * TRUE if batch successfully completed.
361 * @param array $results
364 function locale_translate_batch_finished($success, array $results) {
365 $logger = \Drupal::logger('locale');
367 $additions = $updates = $deletes = $skips = $config = 0;
368 if (isset($results['failed_files'])) {
369 if (\Drupal::moduleHandler()->moduleExists('dblog') && \Drupal::currentUser()->hasPermission('access site reports')) {
370 $message = \Drupal::translation()->formatPlural(count($results['failed_files']), 'One translation file could not be imported. <a href=":url">See the log</a> for details.', '@count translation files could not be imported. <a href=":url">See the log</a> for details.', [':url' => \Drupal::url('dblog.overview')]);
373 $message = \Drupal::translation()->formatPlural(count($results['failed_files']), 'One translation file could not be imported. See the log for details.', '@count translation files could not be imported. See the log for details.');
375 drupal_set_message($message, 'error');
377 if (isset($results['files'])) {
379 // If there are no results and/or no stats (eg. coping with an empty .po
380 // file), simply do nothing.
381 if ($results && isset($results['stats'])) {
382 foreach ($results['stats'] as $filepath => $report) {
383 $additions += $report['additions'];
384 $updates += $report['updates'];
385 $deletes += $report['deletes'];
386 $skips += $report['skips'];
387 if ($report['skips'] > 0) {
388 $skipped_files[] = $filepath;
392 drupal_set_message(\Drupal::translation()->formatPlural(count($results['files']),
393 'One translation file imported. %number translations were added, %update translations were updated and %delete translations were removed.',
394 '@count translation files imported. %number translations were added, %update translations were updated and %delete translations were removed.',
395 ['%number' => $additions, '%update' => $updates, '%delete' => $deletes]
397 $logger->notice('Translations imported: %number added, %update updated, %delete removed.', ['%number' => $additions, '%update' => $updates, '%delete' => $deletes]);
400 if (\Drupal::moduleHandler()->moduleExists('dblog') && \Drupal::currentUser()->hasPermission('access site reports')) {
401 $message = \Drupal::translation()->formatPlural($skips, 'One translation string was skipped because of disallowed or malformed HTML. <a href=":url">See the log</a> for details.', '@count translation strings were skipped because of disallowed or malformed HTML. <a href=":url">See the log</a> for details.', [':url' => \Drupal::url('dblog.overview')]);
404 $message = \Drupal::translation()->formatPlural($skips, 'One translation string was skipped because of disallowed or malformed HTML. See the log for details.', '@count translation strings were skipped because of disallowed or malformed HTML. See the log for details.');
406 drupal_set_message($message, 'warning');
407 $logger->warning('@count disallowed HTML string(s) in files: @files.', ['@count' => $skips, '@files' => implode(',', $skipped_files)]);
411 // Add messages for configuration too.
412 if (isset($results['stats']['config'])) {
413 locale_config_batch_finished($success, $results);
418 * Creates a file object and populates the timestamp property.
420 * @param string $filepath
421 * The filepath of a file to import.
424 * An object representing the file.
426 function locale_translate_file_create($filepath) {
427 $file = new stdClass();
428 $file->filename = drupal_basename($filepath);
429 $file->uri = $filepath;
430 $file->timestamp = filemtime($file->uri);
435 * Generates file properties from filename and options.
437 * An attempt is made to determine the translation language, project name and
438 * project version from the file name. Supported file name patterns are:
439 * {project}-{version}.{langcode}.po, {prefix}.{langcode}.po or {langcode}.po.
440 * Alternatively the translation language can be set using the $options.
442 * @param object $file
443 * A file object of the gettext file to be imported.
444 * @param array $options
445 * An array with options:
446 * - 'langcode': The language code. Overrides the file language.
449 * Modified file object.
451 function locale_translate_file_attach_properties($file, array $options = []) {
452 // If $file is a file entity, convert it to a stdClass.
453 if ($file instanceof FileInterface) {
455 'filename' => $file->getFilename(),
456 'uri' => $file->getFileUri(),
460 // Extract project, version and language code from the file name. Supported:
461 // {project}-{version}.{langcode}.po, {prefix}.{langcode}.po or {langcode}.po
463 ( # project OR project and version OR empty (group 1)
464 ([a-z_]+) # project name (group 2)
467 ([a-z_]+) # project name (group 3)
469 ([0-9a-z\.\-\+]+) # version (group 4)
473 ([^\./]+) # language code (group 5)
476 $!x', $file->filename, $matches);
477 if (isset($matches[5])) {
478 $file->project = $matches[2] . $matches[3];
479 $file->version = $matches[4];
480 $file->langcode = isset($options['langcode']) ? $options['langcode'] : $matches[5];
483 $file->langcode = LanguageInterface::LANGCODE_NOT_SPECIFIED;
489 * Deletes interface translation files and translation history records.
491 * @param array $projects
492 * (optional) Project names from which to delete the translation files and
493 * history. Defaults to all projects.
494 * @param array $langcodes
495 * (optional) Language codes from which to delete the translation files and
496 * history. Defaults to all languages.
499 * TRUE if files are removed successfully. FALSE if one or more files could
502 function locale_translate_delete_translation_files(array $projects = [], array $langcodes = []) {
504 locale_translation_file_history_delete($projects, $langcodes);
506 // Delete all translation files from the translations directory.
507 if ($files = locale_translate_get_interface_translation_files($projects, $langcodes)) {
508 foreach ($files as $file) {
509 $success = file_unmanaged_delete($file->uri);
519 * Builds a locale batch to refresh configuration.
521 * @param array $options
522 * An array with options that can have the following elements:
523 * - 'finish_feedback': (optional) Whether or not to give feedback to the user
524 * when the batch is finished. Defaults to TRUE.
525 * @param array $langcodes
526 * (optional) Array of language codes. Defaults to all translatable languages.
527 * @param array $components
528 * (optional) Array of component lists indexed by type. If not present or it
529 * is an empty array, it will update all components.
532 * The batch definition.
534 function locale_config_batch_update_components(array $options, array $langcodes = [], array $components = []) {
535 $langcodes = $langcodes ? $langcodes : array_keys(\Drupal::languageManager()->getLanguages());
536 if ($langcodes && $names = Locale::config()->getComponentNames($components)) {
537 return locale_config_batch_build($names, $langcodes, $options);
542 * Creates a locale batch to refresh specific configuration.
544 * @param array $names
545 * List of configuration object names (which are strings) to update.
546 * @param array $langcodes
547 * List of language codes to refresh.
548 * @param array $options
549 * (optional) An array with options that can have the following elements:
550 * - 'finish_feedback': Whether or not to give feedback to the user when the
551 * batch is finished. Defaults to TRUE.
554 * The batch definition.
556 * @see locale_config_batch_refresh_name()
558 function locale_config_batch_build(array $names, array $langcodes, array $options = []) {
559 $options += ['finish_feedback' => TRUE];
563 foreach ($names as $name) {
564 $batch_names[] = $name;
566 // During installation the caching of configuration objects is disabled so
567 // it is very expensive to initialize the \Drupal::config() object on each
568 // request. We batch a small number of configuration object upgrades
569 // together to improve the overall performance of the process.
571 $operations[] = ['locale_config_batch_refresh_name', [$batch_names, $langcodes]];
575 if (!empty($batch_names)) {
576 $operations[] = ['locale_config_batch_refresh_name', [$batch_names, $langcodes]];
579 'operations' => $operations,
580 'title' => t('Updating configuration translations'),
581 'init_message' => t('Starting configuration update'),
582 'error_message' => t('Error updating configuration translations'),
583 'file' => drupal_get_path('module', 'locale') . '/locale.bulk.inc',
585 if (!empty($options['finish_feedback'])) {
586 $batch['completed'] = 'locale_config_batch_finished';
592 * Implements callback_batch_operation().
594 * Performs configuration translation refresh.
596 * @param array $names
597 * An array of names of configuration objects to update.
598 * @param array $langcodes
599 * (optional) Array of language codes to update. Defaults to all languages.
600 * @param array|\ArrayAccess $context
601 * Contains a list of files imported.
603 * @see locale_config_batch_build()
605 function locale_config_batch_refresh_name(array $names, array $langcodes, &$context) {
606 if (!isset($context['result']['stats']['config'])) {
607 $context['result']['stats']['config'] = 0;
609 $context['result']['stats']['config'] += Locale::config()->updateConfigTranslations($names, $langcodes);
610 foreach ($names as $name) {
611 $context['result']['names'][] = $name;
613 $context['result']['langcodes'] = $langcodes;
614 $context['finished'] = 1;
618 * Implements callback_batch_finished().
620 * Finishes callback of system page locale import batch.
622 * @param bool $success
623 * Information about the success of the batch import.
624 * @param array $results
625 * Information about the results of the batch import.
627 * @see locale_config_batch_build()
629 function locale_config_batch_finished($success, array $results) {
631 $configuration = isset($results['stats']['config']) ? $results['stats']['config'] : 0;
632 if ($configuration) {
633 drupal_set_message(t('The configuration was successfully updated. There are %number configuration objects updated.', ['%number' => $configuration]));
634 \Drupal::logger('locale')->notice('The configuration was successfully updated. %number configuration objects updated.', ['%number' => $configuration]);
637 drupal_set_message(t('No configuration objects have been updated.'));
638 \Drupal::logger('locale')->warning('No configuration objects have been updated.');