i18n_subsites.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445
  1. """i18n_subsites plugin creates i18n-ized subsites of the default site
  2. This plugin is designed for Pelican 3.4 and later
  3. """
  4. import os
  5. import six
  6. import logging
  7. import posixpath
  8. from copy import copy
  9. from itertools import chain
  10. from operator import attrgetter
  11. from collections import OrderedDict
  12. from contextlib import contextmanager
  13. from six.moves.urllib.parse import urlparse
  14. import gettext
  15. import locale
  16. from pelican import signals
  17. from pelican.generators import ArticlesGenerator, PagesGenerator
  18. from pelican.settings import configure_settings
  19. from pelican.contents import Draft
  20. # Global vars
  21. _MAIN_SETTINGS = None # settings dict of the main Pelican instance
  22. _MAIN_LANG = None # lang of the main Pelican instance
  23. _MAIN_SITEURL = None # siteurl of the main Pelican instance
  24. _MAIN_STATIC_FILES = None # list of Static instances the main Pelican instance
  25. _SUBSITE_QUEUE = {} # map: lang -> settings overrides
  26. _SITE_DB = OrderedDict() # OrderedDict: lang -> siteurl
  27. _SITES_RELPATH_DB = {} # map: (lang, base_lang) -> relpath
  28. # map: generator -> list of removed contents that need interlinking
  29. _GENERATOR_DB = {}
  30. _NATIVE_CONTENT_URL_DB = {} # map: source_path -> content in its native lang
  31. _LOGGER = logging.getLogger(__name__)
  32. @contextmanager
  33. def temporary_locale(temp_locale=None):
  34. '''Enable code to run in a context with a temporary locale
  35. Resets the locale back when exiting context.
  36. Can set a temporary locale if provided
  37. '''
  38. orig_locale = locale.setlocale(locale.LC_ALL)
  39. if temp_locale is not None:
  40. locale.setlocale(locale.LC_ALL, temp_locale)
  41. yield
  42. locale.setlocale(locale.LC_ALL, orig_locale)
  43. def initialize_dbs(settings):
  44. '''Initialize internal DBs using the Pelican settings dict
  45. This clears the DBs for e.g. autoreload mode to work
  46. '''
  47. global _MAIN_SETTINGS, _MAIN_SITEURL, _MAIN_LANG, _SUBSITE_QUEUE
  48. _MAIN_SETTINGS = settings
  49. _MAIN_LANG = settings['DEFAULT_LANG']
  50. _MAIN_SITEURL = settings['SITEURL']
  51. _SUBSITE_QUEUE = settings.get('I18N_SUBSITES', {}).copy()
  52. prepare_site_db_and_overrides()
  53. # clear databases in case of autoreload mode
  54. _SITES_RELPATH_DB.clear()
  55. _NATIVE_CONTENT_URL_DB.clear()
  56. _GENERATOR_DB.clear()
  57. def prepare_site_db_and_overrides():
  58. '''Prepare overrides and create _SITE_DB
  59. _SITE_DB.keys() need to be ready for filter_translations
  60. '''
  61. _SITE_DB.clear()
  62. _SITE_DB[_MAIN_LANG] = _MAIN_SITEURL
  63. # make sure it works for both root-relative and absolute
  64. main_siteurl = '/' if _MAIN_SITEURL == '' else _MAIN_SITEURL
  65. for lang, overrides in _SUBSITE_QUEUE.items():
  66. if 'SITEURL' not in overrides:
  67. overrides['SITEURL'] = posixpath.join(main_siteurl, lang)
  68. _SITE_DB[lang] = overrides['SITEURL']
  69. # default subsite hierarchy
  70. if 'OUTPUT_PATH' not in overrides:
  71. overrides['OUTPUT_PATH'] = os.path.join(
  72. _MAIN_SETTINGS['OUTPUT_PATH'], lang)
  73. if 'CACHE_PATH' not in overrides:
  74. overrides['CACHE_PATH'] = os.path.join(
  75. _MAIN_SETTINGS['CACHE_PATH'], lang)
  76. if 'STATIC_PATHS' not in overrides:
  77. overrides['STATIC_PATHS'] = []
  78. if ('THEME' not in overrides and 'THEME_STATIC_DIR' not in overrides and
  79. 'THEME_STATIC_PATHS' not in overrides):
  80. relpath = relpath_to_site(lang, _MAIN_LANG)
  81. overrides['THEME_STATIC_DIR'] = posixpath.join(
  82. relpath, _MAIN_SETTINGS['THEME_STATIC_DIR'])
  83. overrides['THEME_STATIC_PATHS'] = []
  84. # to change what is perceived as translations
  85. overrides['DEFAULT_LANG'] = lang
  86. def subscribe_filter_to_signals(settings):
  87. '''Subscribe content filter to requested signals'''
  88. for sig in settings.get('I18N_FILTER_SIGNALS', []):
  89. sig.connect(filter_contents_translations)
  90. def initialize_plugin(pelican_obj):
  91. '''Initialize plugin variables and Pelican settings'''
  92. if _MAIN_SETTINGS is None:
  93. initialize_dbs(pelican_obj.settings)
  94. subscribe_filter_to_signals(pelican_obj.settings)
  95. def get_site_path(url):
  96. '''Get the path component of an url, excludes siteurl
  97. also normalizes '' to '/' for relpath to work,
  98. otherwise it could be interpreted as a relative filesystem path
  99. '''
  100. path = urlparse(url).path
  101. if path == '':
  102. path = '/'
  103. return path
  104. def relpath_to_site(lang, target_lang):
  105. '''Get relative path from siteurl of lang to siteurl of base_lang
  106. the output is cached in _SITES_RELPATH_DB
  107. '''
  108. path = _SITES_RELPATH_DB.get((lang, target_lang), None)
  109. if path is None:
  110. siteurl = _SITE_DB.get(lang, _MAIN_SITEURL)
  111. target_siteurl = _SITE_DB.get(target_lang, _MAIN_SITEURL)
  112. path = posixpath.relpath(get_site_path(target_siteurl),
  113. get_site_path(siteurl))
  114. _SITES_RELPATH_DB[(lang, target_lang)] = path
  115. return path
  116. def save_generator(generator):
  117. '''Save the generator for later use
  118. initialize the removed content list
  119. '''
  120. _GENERATOR_DB[generator] = []
  121. def article2draft(article):
  122. '''Transform an Article to Draft'''
  123. draft = Draft(article._content, article.metadata, article.settings,
  124. article.source_path, article._context)
  125. draft.status = 'draft'
  126. return draft
  127. def page2hidden_page(page):
  128. '''Transform a Page to a hidden Page'''
  129. page.status = 'hidden'
  130. return page
  131. class GeneratorInspector(object):
  132. '''Inspector of generator instances'''
  133. generators_info = {
  134. ArticlesGenerator: {
  135. 'translations_lists': ['translations', 'drafts_translations'],
  136. 'contents_lists': [('articles', 'drafts')],
  137. 'hiding_func': article2draft,
  138. 'policy': 'I18N_UNTRANSLATED_ARTICLES',
  139. },
  140. PagesGenerator: {
  141. 'translations_lists': ['translations', 'hidden_translations'],
  142. 'contents_lists': [('pages', 'hidden_pages')],
  143. 'hiding_func': page2hidden_page,
  144. 'policy': 'I18N_UNTRANSLATED_PAGES',
  145. },
  146. }
  147. def __init__(self, generator):
  148. '''Identify the best known class of the generator instance
  149. The class '''
  150. self.generator = generator
  151. self.generators_info.update(generator.settings.get(
  152. 'I18N_GENERATORS_INFO', {}))
  153. for cls in generator.__class__.__mro__:
  154. if cls in self.generators_info:
  155. self.info = self.generators_info[cls]
  156. break
  157. else:
  158. self.info = {}
  159. def translations_lists(self):
  160. '''Iterator over lists of content translations'''
  161. return (getattr(self.generator, name) for name in
  162. self.info.get('translations_lists', []))
  163. def contents_list_pairs(self):
  164. '''Iterator over pairs of normal and hidden contents'''
  165. return (tuple(getattr(self.generator, name) for name in names)
  166. for names in self.info.get('contents_lists', []))
  167. def hiding_function(self):
  168. '''Function for transforming content to a hidden version'''
  169. hiding_func = self.info.get('hiding_func', lambda x: x)
  170. return hiding_func
  171. def untranslated_policy(self, default):
  172. '''Get the policy for untranslated content'''
  173. return self.generator.settings.get(self.info.get('policy', None),
  174. default)
  175. def all_contents(self):
  176. '''Iterator over all contents'''
  177. translations_iterator = chain(*self.translations_lists())
  178. return chain(translations_iterator,
  179. *(pair[i] for pair in self.contents_list_pairs()
  180. for i in (0, 1)))
  181. def filter_contents_translations(generator):
  182. '''Filter the content and translations lists of a generator
  183. Filters out
  184. 1) translations which will be generated in a different site
  185. 2) content that is not in the language of the currently
  186. generated site but in that of a different site, content in a
  187. language which has no site is generated always. The filtering
  188. method bay be modified by the respective untranslated policy
  189. '''
  190. inspector = GeneratorInspector(generator)
  191. current_lang = generator.settings['DEFAULT_LANG']
  192. langs_with_sites = _SITE_DB.keys()
  193. removed_contents = _GENERATOR_DB[generator]
  194. for translations in inspector.translations_lists():
  195. for translation in translations[:]: # copy to be able to remove
  196. if translation.lang in langs_with_sites:
  197. translations.remove(translation)
  198. removed_contents.append(translation)
  199. hiding_func = inspector.hiding_function()
  200. untrans_policy = inspector.untranslated_policy(default='hide')
  201. for (contents, other_contents) in inspector.contents_list_pairs():
  202. for content in other_contents: # save any hidden native content first
  203. if content.lang == current_lang: # in native lang
  204. # save the native URL attr formatted in the current locale
  205. _NATIVE_CONTENT_URL_DB[content.source_path] = content.url
  206. for content in contents[:]: # copy for removing in loop
  207. if content.lang == current_lang: # in native lang
  208. # save the native URL attr formatted in the current locale
  209. _NATIVE_CONTENT_URL_DB[content.source_path] = content.url
  210. elif content.lang in langs_with_sites and untrans_policy != 'keep':
  211. contents.remove(content)
  212. if untrans_policy == 'hide':
  213. other_contents.append(hiding_func(content))
  214. elif untrans_policy == 'remove':
  215. removed_contents.append(content)
  216. def install_templates_translations(generator):
  217. '''Install gettext translations in the jinja2.Environment
  218. Only if the 'jinja2.ext.i18n' jinja2 extension is enabled
  219. the translations for the current DEFAULT_LANG are installed.
  220. '''
  221. if 'jinja2.ext.i18n' in generator.settings['JINJA_EXTENSIONS']:
  222. domain = generator.settings.get('I18N_GETTEXT_DOMAIN', 'messages')
  223. localedir = generator.settings.get('I18N_GETTEXT_LOCALEDIR')
  224. if localedir is None:
  225. localedir = os.path.join(generator.theme, 'translations')
  226. current_lang = generator.settings['DEFAULT_LANG']
  227. if current_lang == generator.settings.get('I18N_TEMPLATES_LANG',
  228. _MAIN_LANG):
  229. translations = gettext.NullTranslations()
  230. else:
  231. langs = [current_lang]
  232. try:
  233. translations = gettext.translation(domain, localedir, langs)
  234. except (IOError, OSError):
  235. _LOGGER.error((
  236. "Cannot find translations for language '{}' in '{}' with "
  237. "domain '{}'. Installing NullTranslations.").format(
  238. langs[0], localedir, domain))
  239. translations = gettext.NullTranslations()
  240. newstyle = generator.settings.get('I18N_GETTEXT_NEWSTYLE', True)
  241. generator.env.install_gettext_translations(translations, newstyle)
  242. def add_variables_to_context(generator):
  243. '''Adds useful iterable variables to template context'''
  244. context = generator.context # minimize attr lookup
  245. context['relpath_to_site'] = relpath_to_site
  246. context['main_siteurl'] = _MAIN_SITEURL
  247. context['main_lang'] = _MAIN_LANG
  248. context['lang_siteurls'] = _SITE_DB
  249. current_lang = generator.settings['DEFAULT_LANG']
  250. extra_siteurls = _SITE_DB.copy()
  251. extra_siteurls.pop(current_lang)
  252. context['extra_siteurls'] = extra_siteurls
  253. def interlink_translations(content):
  254. '''Link content to translations in their main language
  255. so the URL (including localized month names) of the different subsites
  256. will be honored
  257. '''
  258. lang = content.lang
  259. # sort translations by lang
  260. content.translations.sort(key=attrgetter('lang'))
  261. for translation in content.translations:
  262. relpath = relpath_to_site(lang, translation.lang)
  263. url = _NATIVE_CONTENT_URL_DB[translation.source_path]
  264. translation.override_url = posixpath.join(relpath, url)
  265. def interlink_translated_content(generator):
  266. '''Make translations link to the native locations
  267. for generators that may contain translated content
  268. '''
  269. inspector = GeneratorInspector(generator)
  270. for content in inspector.all_contents():
  271. interlink_translations(content)
  272. def interlink_removed_content(generator):
  273. '''For all contents removed from generation queue update interlinks
  274. link to the native location
  275. '''
  276. current_lang = generator.settings['DEFAULT_LANG']
  277. for content in _GENERATOR_DB[generator]:
  278. url = _NATIVE_CONTENT_URL_DB[content.source_path]
  279. relpath = relpath_to_site(current_lang, content.lang)
  280. content.override_url = posixpath.join(relpath, url)
  281. def interlink_static_files(generator):
  282. '''Add links to static files in the main site if necessary'''
  283. if generator.settings['STATIC_PATHS'] != []:
  284. return # customized STATIC_PATHS
  285. filenames = generator.context['filenames'] # minimize attr lookup
  286. relpath = relpath_to_site(generator.settings['DEFAULT_LANG'], _MAIN_LANG)
  287. for staticfile in _MAIN_STATIC_FILES:
  288. if staticfile.get_relative_source_path() not in filenames:
  289. staticfile = copy(staticfile) # prevent override in main site
  290. staticfile.override_url = posixpath.join(relpath, staticfile.url)
  291. generator.add_source_path(staticfile)
  292. def save_main_static_files(static_generator):
  293. '''Save the static files generated for the main site'''
  294. global _MAIN_STATIC_FILES
  295. # test just for current lang as settings change in autoreload mode
  296. if static_generator.settings['DEFAULT_LANG'] == _MAIN_LANG:
  297. _MAIN_STATIC_FILES = static_generator.staticfiles
  298. def update_generators():
  299. '''Update the context of all generators
  300. Ads useful variables and translations into the template context
  301. and interlink translations
  302. '''
  303. for generator in _GENERATOR_DB.keys():
  304. install_templates_translations(generator)
  305. add_variables_to_context(generator)
  306. interlink_static_files(generator)
  307. interlink_removed_content(generator)
  308. interlink_translated_content(generator)
  309. def get_pelican_cls(settings):
  310. '''Get the Pelican class requested in settings'''
  311. cls = settings['PELICAN_CLASS']
  312. if isinstance(cls, six.string_types):
  313. module, cls_name = cls.rsplit('.', 1)
  314. module = __import__(module)
  315. cls = getattr(module, cls_name)
  316. return cls
  317. def create_next_subsite(pelican_obj):
  318. '''Create the next subsite using the lang-specific config
  319. If there are no more subsites in the generation queue, update all
  320. the generators (interlink translations and removed content, add
  321. variables and translations to template context). Otherwise get the
  322. language and overrides for next the subsite in the queue and apply
  323. overrides. Then generate the subsite using a PELICAN_CLASS
  324. instance and its run method. Finally, restore the previous locale.
  325. '''
  326. global _MAIN_SETTINGS
  327. if len(_SUBSITE_QUEUE) == 0:
  328. _LOGGER.debug(
  329. 'i18n: Updating cross-site links and context of all generators.')
  330. update_generators()
  331. _MAIN_SETTINGS = None # to initialize next time
  332. else:
  333. with temporary_locale():
  334. settings = _MAIN_SETTINGS.copy()
  335. lang, overrides = _SUBSITE_QUEUE.popitem()
  336. settings.update(overrides)
  337. settings = configure_settings(settings) # to set LOCALE, etc.
  338. cls = get_pelican_cls(settings)
  339. new_pelican_obj = cls(settings)
  340. _LOGGER.debug(("Generating i18n subsite for language '{}' "
  341. "using class {}").format(lang, cls))
  342. new_pelican_obj.run()
  343. # map: signal name -> function name
  344. _SIGNAL_HANDLERS_DB = {
  345. 'get_generators': initialize_plugin,
  346. 'article_generator_pretaxonomy': filter_contents_translations,
  347. 'page_generator_finalized': filter_contents_translations,
  348. 'get_writer': create_next_subsite,
  349. 'static_generator_finalized': save_main_static_files,
  350. 'generator_init': save_generator,
  351. }
  352. def register():
  353. '''Register the plugin only if required signals are available'''
  354. for sig_name in _SIGNAL_HANDLERS_DB.keys():
  355. if not hasattr(signals, sig_name):
  356. _LOGGER.error((
  357. 'The i18n_subsites plugin requires the {} '
  358. 'signal available for sure in Pelican 3.4.0 and later, '
  359. 'plugin will not be used.').format(sig_name))
  360. return
  361. for sig_name, handler in _SIGNAL_HANDLERS_DB.items():
  362. sig = getattr(signals, sig_name)
  363. sig.connect(handler)