parse_page.js 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510
  1. import { load as cheerioLoad } from 'cheerio';
  2. import { MessageEmbed } from 'discord.js';
  3. import { toSection } from '../util/wiki.js';
  4. import { got, parse_infobox, htmlToPlain, htmlToDiscord, escapeFormatting, limitLength } from '../util/functions.js';
  5. const parsedContentModels = [
  6. 'wikitext',
  7. 'wikibase-item',
  8. 'wikibase-lexeme',
  9. 'wikibase-property'
  10. ];
  11. // Max length of 10 characters
  12. const contentModels = {
  13. Scribunto: 'lua',
  14. javascript: 'js',
  15. json: 'json',
  16. css: 'css'
  17. };
  18. const contentFormats = {
  19. 'application/json': 'json',
  20. 'text/javascript': 'js',
  21. 'text/css': 'css'
  22. };
  23. // Max length of 10 characters
  24. const infoboxList = [
  25. '.infobox',
  26. '.portable-infobox',
  27. '.infoboxtable',
  28. '.notaninfobox',
  29. '.tpl-infobox'
  30. ];
  31. const removeClasses = [
  32. 'table',
  33. 'figure',
  34. 'script',
  35. 'input',
  36. 'style',
  37. 'script',
  38. 'noscript',
  39. 'ul.gallery',
  40. '.mw-editsection',
  41. 'sup.reference',
  42. 'ol.references',
  43. '.thumb',
  44. '.error',
  45. '.nomobile',
  46. '.noprint',
  47. '.noexcerpt',
  48. '.sortkey',
  49. '.mw-collapsible.mw-collapsed',
  50. '.c-item-hoverbox__display',
  51. 'wb\\:sectionedit'
  52. ];
  53. const removeClassesExceptions = [
  54. 'div.main-page-tag-lcs',
  55. 'div.lcs-container',
  56. 'div.mw-highlight',
  57. 'div.poem',
  58. 'div.hlist',
  59. 'div.treeview',
  60. 'div.redirectMsg',
  61. 'div.introduction',
  62. 'div.wikibase-entityview',
  63. 'div.wikibase-entityview-main',
  64. 'div.wikibase-entitytermsview',
  65. 'div.wikibase-entitytermsview-heading',
  66. 'div.wikibase-entitytermsview-heading-description',
  67. 'div#wb-lexeme-header',
  68. 'div#wb-lexeme-header div:not([class]):not([id])',
  69. 'div.language-lexical-category-widget'
  70. ];
  71. /**
  72. * Parses a wiki page to get it's description.
  73. * @param {import('../util/i18n.js').default} lang - The user language.
  74. * @param {import('discord.js').Message} msg - The Discord message.
  75. * @param {String} content - The content for the message.
  76. * @param {import('discord.js').MessageEmbed} embed - The embed for the message.
  77. * @param {import('../util/wiki.js').default} wiki - The wiki for the page.
  78. * @param {import('discord.js').MessageReaction} reaction - The reaction on the message.
  79. * @param {Object} querypage - The details of the page.
  80. * @param {Number} querypage.ns - The namespace of the page.
  81. * @param {String} querypage.title - The title of the page.
  82. * @param {String} querypage.contentmodel - The content model of the page.
  83. * @param {String} querypage.pagelanguage - The language of the page.
  84. * @param {String} [querypage.missing] - If the page doesn't exist.
  85. * @param {String} [querypage.known] - If the page is known.
  86. * @param {Object} [querypage.pageprops] - The properties of the page.
  87. * @param {String} [querypage.pageprops.infoboxes] - The JSON data for portable infoboxes on the page.
  88. * @param {String} [querypage.pageprops.disambiguation] - The disambiguation property of the page.
  89. * @param {String} [querypage.pageprops.uselang] - The language of the page description.
  90. * @param {Boolean} [querypage.pageprops.noRedirect] - If the page is allowed to be redirected.
  91. * @param {String} [thumbnail] - The default thumbnail for the wiki.
  92. * @param {String} [fragment] - The section title to embed.
  93. * @param {String} [pagelink] - The link to the page.
  94. * @returns {Promise<import('discord.js').Message>} The edited message.
  95. */
  96. export default function parse_page(lang, msg, content, embed, wiki, reaction, {ns, title, contentmodel, pagelanguage, missing, known, pageprops: {infoboxes, disambiguation} = {}, uselang = lang.lang, noRedirect = false}, thumbnail = '', fragment = '', pagelink = '') {
  97. if ( reaction ) reaction.removeEmoji();
  98. if ( !msg?.showEmbed?.() || ( missing !== undefined && ( ns !== 8 || known === undefined ) ) || !embed || embed.description ) {
  99. if ( missing !== undefined && embed ) {
  100. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  101. embed.spliceFields( 0, 0, embed.backupField );
  102. }
  103. if ( embed.backupDescription && embed.length < 5000 ) {
  104. embed.setDescription( embed.backupDescription );
  105. }
  106. }
  107. return msg.sendChannel( {content: content, embeds: [embed]} );
  108. }
  109. return msg.sendChannel( {
  110. content,
  111. embeds: [new MessageEmbed(embed).setDescription( '<a:loading:641343250661113886> **' + lang.get('search.loading') + '**' )]
  112. } ).then( message => {
  113. if ( !message ) return;
  114. if ( ns === 8 ) {
  115. title = title.split(':').slice(1).join(':');
  116. if ( title.endsWith( '/' + pagelanguage ) ) title = title.substring(0, title.length - ( pagelanguage.length + 1 ));
  117. return got.get( wiki + 'api.php?action=query&meta=allmessages&amprop=default&amincludelocal=true&amlang=' + encodeURIComponent( pagelanguage ) + '&ammessages=' + encodeURIComponent( title ) + '&format=json', {
  118. timeout: {
  119. request: 10_000
  120. }
  121. } ).then( response => {
  122. var body = response.body;
  123. if ( body && body.warnings ) log_warning(body.warnings);
  124. if ( response.statusCode !== 200 || !body || body.batchcomplete === undefined || !body.query?.allmessages?.[0] ) {
  125. console.log( '- ' + response.statusCode + ': Error while getting the system message: ' + body?.error?.info );
  126. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  127. embed.spliceFields( 0, 0, embed.backupField );
  128. }
  129. if ( embed.backupDescription && embed.length < 5000 ) {
  130. embed.setDescription( embed.backupDescription );
  131. }
  132. return;
  133. }
  134. if ( !embed.description && embed.length < 4000 ) {
  135. var description = body.query.allmessages[0]['*'];
  136. var regex = /^L(\d+)(?:-L?(\d+))?$/.exec(fragment);
  137. if ( regex ) {
  138. let descArray = description.split('\n').slice(regex[1] - 1, ( regex[2] || regex[1] ));
  139. if ( descArray.length ) {
  140. description = descArray.join('\n').replace( /^\n+/, '' ).replace( /\n+$/, '' );
  141. if ( description ) {
  142. if ( description.length > 2000 ) description = description.substring(0, 2000) + '\u2026';
  143. description = '```' + ( contentModels[contentmodel] || '' ) + '\n' + description + '\n```';
  144. embed.setDescription( description );
  145. }
  146. }
  147. }
  148. else {
  149. let defaultDescription = body.query.allmessages[0].default;
  150. if ( description.trim() ) {
  151. description = description.replace( /^\n+/, '' ).replace( /\n+$/, '' );
  152. if ( description.length > 500 ) description = description.substring(0, 500) + '\u2026';
  153. description = '```' + ( contentModels[contentmodel] || '' ) + '\n' + description + '\n```';
  154. embed.setDescription( description );
  155. }
  156. else if ( embed.backupDescription ) {
  157. embed.setDescription( embed.backupDescription );
  158. }
  159. if ( defaultDescription?.trim() ) {
  160. defaultDescription = defaultDescription.replace( /^\n+/, '' ).replace( /\n+$/, '' );
  161. if ( defaultDescription.length > 250 ) defaultDescription = defaultDescription.substring(0, 250) + '\u2026';
  162. defaultDescription = '```' + ( contentModels[contentmodel] || '' ) + '\n' + defaultDescription + '\n```';
  163. embed.addField( lang.get('search.messagedefault'), defaultDescription );
  164. }
  165. else if ( body.query.allmessages[0].defaultmissing !== undefined ) {
  166. embed.addField( lang.get('search.messagedefault'), lang.get('search.messagedefaultnone') );
  167. }
  168. }
  169. }
  170. }, error => {
  171. console.log( '- Error while getting the system message: ' + error );
  172. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  173. embed.spliceFields( 0, 0, embed.backupField );
  174. }
  175. if ( embed.backupDescription && embed.length < 5000 ) {
  176. embed.setDescription( embed.backupDescription );
  177. }
  178. } ).then( () => {
  179. return message.edit( {content, embeds: [embed]} ).catch(log_error);
  180. } );
  181. }
  182. if ( !parsedContentModels.includes( contentmodel ) ) return got.get( wiki + 'api.php?action=query&prop=revisions&rvprop=content&rvslots=main&converttitles=true&titles=%1F' + encodeURIComponent( title ) + '&format=json', {
  183. timeout: {
  184. request: 10_000
  185. }
  186. } ).then( response => {
  187. var body = response.body;
  188. if ( body && body.warnings ) log_warning(body.warnings);
  189. var revision = Object.values(( body?.query?.pages || {} ))?.[0]?.revisions?.[0];
  190. revision = ( revision?.slots?.main || revision );
  191. if ( response.statusCode !== 200 || !body || body.batchcomplete === undefined || !revision?.['*'] ) {
  192. console.log( '- ' + response.statusCode + ': Error while getting the page content: ' + ( body && body.error && body.error.info ) );
  193. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  194. embed.spliceFields( 0, 0, embed.backupField );
  195. }
  196. if ( embed.backupDescription && embed.length < 5000 ) {
  197. embed.setDescription( embed.backupDescription );
  198. }
  199. return;
  200. }
  201. if ( !embed.description && embed.length < 4000 ) {
  202. var description = revision['*'];
  203. var regex = /^L(\d+)(?:-L?(\d+))?$/.exec(fragment);
  204. if ( regex ) {
  205. let descArray = description.split('\n').slice(regex[1] - 1, ( regex[2] || regex[1] ));
  206. if ( descArray.length ) {
  207. description = descArray.join('\n').replace( /^\n+/, '' ).replace( /\n+$/, '' );
  208. if ( description ) {
  209. if ( description.length > 2000 ) description = description.substring(0, 2000) + '\u2026';
  210. description = '```' + ( contentModels[revision.contentmodel] || contentFormats[revision.contentformat] || '' ) + '\n' + description + '\n```';
  211. embed.setDescription( description );
  212. }
  213. }
  214. }
  215. else if ( description.trim() ) {
  216. description = description.replace( /^\n+/, '' ).replace( /\n+$/, '' );
  217. if ( description.length > 500 ) description = description.substring(0, 500) + '\u2026';
  218. description = '```' + ( contentModels[revision.contentmodel] || contentFormats[revision.contentformat] || '' ) + '\n' + description + '\n```';
  219. embed.setDescription( description );
  220. }
  221. else if ( embed.backupDescription ) {
  222. embed.setDescription( embed.backupDescription );
  223. }
  224. }
  225. }, error => {
  226. console.log( '- Error while getting the page content: ' + error );
  227. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  228. embed.spliceFields( 0, 0, embed.backupField );
  229. }
  230. if ( embed.backupDescription && embed.length < 5000 ) {
  231. embed.setDescription( embed.backupDescription );
  232. }
  233. } ).then( () => {
  234. return message.edit( {content, embeds: [embed]} ).catch(log_error);
  235. } );
  236. if ( !fragment && !embed.fields.length && infoboxes ) {
  237. try {
  238. var infobox = JSON.parse(infoboxes)?.[0];
  239. parse_infobox(infobox, embed, thumbnail, embed.url);
  240. }
  241. catch ( error ) {
  242. console.log( '- Failed to parse the infobox: ' + error );
  243. }
  244. }
  245. let extraImages = [];
  246. return got.get( wiki + 'api.php?uselang=' + uselang + '&action=parse' + ( noRedirect ? '' : '&redirects=true' ) + '&prop=text|images|displaytitle' + ( contentmodel !== 'wikitext' || fragment || disambiguation !== undefined ? '' : '&section=0' ) + '&disablelimitreport=true&disableeditsection=true&disabletoc=true&sectionpreview=true&page=' + encodeURIComponent( title ) + '&format=json', {
  247. timeout: {
  248. request: 10_000
  249. }
  250. } ).then( response => {
  251. if ( response.statusCode !== 200 || !response?.body?.parse?.text ) {
  252. console.log( '- ' + response.statusCode + ': Error while parsing the page: ' + response?.body?.error?.info );
  253. if ( embed.backupDescription && embed.length < 5000 ) {
  254. embed.setDescription( embed.backupDescription );
  255. }
  256. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  257. embed.spliceFields( 0, 0, embed.backupField );
  258. }
  259. return;
  260. }
  261. if ( !embed.forceTitle ) {
  262. var displaytitle = htmlToDiscord( response.body.parse.displaytitle );
  263. if ( displaytitle.length > 250 ) displaytitle = displaytitle.substring(0, 250) + '\u2026';
  264. embed.setTitle( displaytitle );
  265. }
  266. var $ = cheerioLoad(response.body.parse.text['*'].replace( /\n?<br(?: ?\/)?>\n?/g, '<br>' ));
  267. if ( embed.brokenInfobox && $('aside.portable-infobox').length ) {
  268. let infobox = $('aside.portable-infobox');
  269. embed.fields.forEach( field => {
  270. if ( embed.length > 5400 ) return;
  271. if ( /^`.+`$/.test(field.name) ) {
  272. let label = infobox.find(field.name.replace( /^`(.+)`$/, '[data-source="$1"] .pi-data-label, .pi-data-label[data-source="$1"]' )).html();
  273. if ( !label ) label = infobox.find(field.name.replace( /^`(.+)`$/, '[data-item-name="$1"] .pi-data-label, .pi-data-label[data-item-name="$1"]' )).html();
  274. if ( label ) {
  275. label = htmlToPlain(label).trim();
  276. if ( label.length > 100 ) label = label.substring(0, 100) + '\u2026';
  277. if ( label ) field.name = label;
  278. }
  279. }
  280. if ( /^`.+`$/.test(field.value) ) {
  281. let value = infobox.find(field.value.replace( /^`(.+)`$/, '[data-source="$1"] .pi-data-value, .pi-data-value[data-source="$1"]' )).html();
  282. if ( !value ) value = infobox.find(field.value.replace( /^`(.+)`$/, '[data-item-name="$1"] .pi-data-value, .pi-data-value[data-item-name="$1"]' )).html();
  283. if ( value ) {
  284. value = htmlToDiscord(value, embed.url).trim().replace( /\n{3,}/g, '\n\n' );
  285. if ( value.length > 500 ) value = limitLength(value, 500, 250);
  286. if ( value ) field.value = value;
  287. }
  288. }
  289. } );
  290. }
  291. if ( !fragment && !embed.fields.length && $(infoboxList.join(', ')).length ) {
  292. let infobox = $(infoboxList.join(', ')).first();
  293. if ( embed.thumbnail?.url === thumbnail ) {
  294. let image = infobox.find([
  295. 'tr:eq(1) img',
  296. 'div.images img',
  297. 'figure.pi-image img',
  298. 'div.infobox-imagearea img'
  299. ].join(', ')).toArray().find( img => {
  300. let imgURL = img.attribs.src;
  301. if ( !imgURL ) return false;
  302. return ( /^(?:https?:)?\/\//.test(imgURL) && /\.(?:png|jpg|jpeg|gif)(?:\/|\?|$)/i.test(imgURL) );
  303. } )?.attribs.src?.replace( /^(?:https?:)?\/\//, 'https://' );
  304. if ( image ) embed.setThumbnail( new URL(image, wiki).href );
  305. }
  306. let rows = infobox.find([
  307. '> tbody > tr',
  308. '> tbody > tr > th.mainheader',
  309. '> tbody > tr > th.infobox-header',
  310. '> table > tbody > tr',
  311. 'div.section > div.title',
  312. 'div.section > table > tbody > tr',
  313. 'h2.pi-header',
  314. 'div.pi-data',
  315. 'table.infobox-rows > tbody > tr',
  316. 'div.infobox-rows:not(.subinfobox) > div.infobox-row'
  317. ].join(', '));
  318. let tdLabel = true;
  319. for ( let i = 0; i < rows.length; i++ ) {
  320. if ( embed.fields.length >= 25 || embed.length > 5400 ) break;
  321. let row = rows.eq(i);
  322. if ( row.is('th.mainheader, th.infobox-header, div.title, h2.pi-header') ) {
  323. row.find(removeClasses.join(', ')).remove();
  324. let label = htmlToDiscord(row, embed.url).trim();
  325. if ( label.length > 100 ) label = label.substring(0, 100) + '\u2026';
  326. if ( label ) {
  327. if ( !label.includes( '**' ) ) label = '**' + label + '**';
  328. if ( embed.fields.length && embed.fields[embed.fields.length - 1].name === '\u200b' ) {
  329. embed.spliceFields( embed.fields.length - 1, 1, {
  330. name: '\u200b',
  331. value: label,
  332. inline: false
  333. } );
  334. }
  335. else embed.addField( '\u200b', label, false );
  336. }
  337. }
  338. else if ( row.is('tr, div.pi-data, div.infobox-row') ) {
  339. let label = row.children(( tdLabel ? 'td, ' : '' ) + 'th, h3.pi-data-label, div.infobox-cell-header').eq(0);
  340. label.find(removeClasses.join(', ')).remove();
  341. let value = row.children('td, div.pi-data-value, div.infobox-cell-data').eq(( label.is('td') ? 1 : 0 ));
  342. value.find(removeClasses.join(', ')).remove();
  343. if ( !label.is('td') && label.html()?.trim() && value.html()?.trim() ) tdLabel = false;
  344. label = htmlToPlain(label).trim().split('\n')[0];
  345. value = htmlToDiscord(value, embed.url).trim().replace( /\n{3,}/g, '\n\n' );
  346. if ( label.length > 100 ) label = label.substring(0, 100) + '\u2026';
  347. if ( value.length > 500 ) value = limitLength(value, 500, 250);
  348. if ( label && value ) embed.addField( label, value, true );
  349. }
  350. }
  351. if ( embed.fields.length && embed.fields[embed.fields.length - 1].name === '\u200b' ) {
  352. embed.spliceFields( embed.fields.length - 1, 1 );
  353. }
  354. }
  355. if ( embed.thumbnail?.url === thumbnail ) {
  356. let image = response.body.parse.images.find( pageimage => ( /\.(?:png|jpg|jpeg|gif)$/.test(pageimage.toLowerCase()) && pageimage.toLowerCase().includes( title.toLowerCase().replace( / /g, '_' ) ) ) );
  357. if ( !image ) {
  358. thumbnail = $(infoboxList.join(', ')).find('img').filter( (i, img) => {
  359. img = $(img).prop('src')?.toLowerCase();
  360. return ( /^(?:https?:)?\/\//.test(img) && /\.(?:png|jpg|jpeg|gif)(?:\/|\?|$)/.test(img) );
  361. } ).first().prop('src');
  362. if ( !thumbnail ) thumbnail = $('img').filter( (i, img) => {
  363. img = $(img).prop('src')?.toLowerCase();
  364. return ( /^(?:https?:)?\/\//.test(img) && /\.(?:png|jpg|jpeg|gif)(?:\/|\?|$)/.test(img) );
  365. } ).first().prop('src');
  366. if ( !thumbnail ) image = response.body.parse.images.find( pageimage => {
  367. return /\.(?:png|jpg|jpeg|gif)$/.test(pageimage.toLowerCase());
  368. } );
  369. }
  370. if ( image ) thumbnail = wiki.toLink('Special:FilePath/' + image);
  371. if ( thumbnail ) embed.setThumbnail( thumbnail.replace( /^(?:https?:)?\/\//, 'https://' ) );
  372. }
  373. if ( fragment && embed.length < 4750 && embed.fields.length < 25 &&
  374. toSection(embed.fields[0]?.name.replace( /^\**_*(.*?)_*\**$/g, '$1' )) !== toSection(fragment) ) {
  375. let newFragment = '';
  376. let exactMatch = true;
  377. let allSections = $('h1, h2, h3, h4, h5, h6').children('span');
  378. var section = allSections.filter( (i, span) => {
  379. return ( '#' + span.attribs.id === toSection(fragment) );
  380. } ).parent();
  381. if ( !section.length ) {
  382. section = $('[id="' + toSection(fragment, false).replace( '#', '' ) + '"]');
  383. newFragment = section.attr('id');
  384. if ( section.is(':empty') ) {
  385. section = section.parent();
  386. if ( ['h1','h2','h3','h4','h5','h6'].includes( section.prev()[0]?.tagName ) ) {
  387. section = section.prev();
  388. if ( section.children('span').first().attr('id') ) {
  389. newFragment = section.children('span').first().attr('id');
  390. }
  391. }
  392. }
  393. }
  394. if ( !section.length ) exactMatch = false;
  395. if ( !section.length ) section = allSections.filter( (i, span) => {
  396. return ( '#' + span.attribs.id?.toLowerCase() === toSection(fragment).toLowerCase() );
  397. } );
  398. if ( !section.length ) section = allSections.filter( (i, span) => {
  399. return ( $(span).parent().text().trim() === fragment );
  400. } );
  401. if ( !section.length ) section = allSections.filter( (i, span) => {
  402. return ( $(span).parent().text().trim().toLowerCase() === fragment.toLowerCase() );
  403. } );
  404. if ( !section.length ) section = allSections.filter( (i, span) => {
  405. return $(span).parent().text().toLowerCase().includes( fragment.toLowerCase() );
  406. } );
  407. if ( !exactMatch && section.length ) {
  408. newFragment = section.attr('id');
  409. section = section.parent();
  410. }
  411. if ( section.length ) {
  412. section = section.first();
  413. var sectionLevel = section[0].tagName.replace('h', '');
  414. if ( !['1','2','3','4','5','6'].includes( sectionLevel ) ) sectionLevel = '10';
  415. var sectionContent = $('<div>').append(
  416. section.nextUntil(['h1','h2','h3','h4','h5','h6'].slice(0, sectionLevel).join(', '))
  417. );
  418. section.find('div, ' + removeClasses.join(', ')).remove();
  419. extraImages.push(...new Set([
  420. ...sectionContent.find(infoboxList.join(', ')).find([
  421. 'tr:eq(1) img',
  422. 'div.images img',
  423. 'figure.pi-image img',
  424. 'div.infobox-imagearea img'
  425. ].join(', ')).toArray(),
  426. ...sectionContent.find([
  427. 'ul.gallery > li.gallerybox img',
  428. 'div.wikia-gallery > div.wikia-gallery-item img',
  429. 'div.ogv-gallery > div.ogv-gallery-item img'
  430. ].join(', ')).toArray()
  431. ].filter( img => {
  432. let imgURL = ( img.attribs.src?.startsWith?.( 'data:' ) ? img.attribs['data-src'] : img.attribs.src );
  433. if ( !imgURL ) return false;
  434. return ( /^(?:https?:)?\/\//.test(imgURL) && /\.(?:png|jpg|jpeg|gif)(?:\/|\?|$)/i.test(imgURL) );
  435. } ).map( img => {
  436. if ( img.attribs['data-image-name']?.endsWith( '.gif' ) ) return wiki.toLink('Special:FilePath/' + img.attribs['data-image-name']);
  437. let imgURL = ( img.attribs.src?.startsWith?.( 'data:' ) ? img.attribs['data-src'] : img.attribs.src );
  438. imgURL = imgURL.replace( /\/thumb(\/[\da-f]\/[\da-f]{2}\/([^\/]+))\/\d+px-\2/, '$1' ).replace( /\/scale-to-width-down\/\d+/, '' );
  439. return new URL(imgURL.replace( /^(?:https?:)?\/\//, 'https://' ), wiki).href;
  440. } )));
  441. sectionContent.find(infoboxList.join(', ')).remove();
  442. sectionContent.find('div, ' + removeClasses.join(', ')).not(removeClassesExceptions.join(', ')).remove();
  443. var name = htmlToPlain(section).trim();
  444. if ( !name.length ) name = escapeFormatting(fragment);
  445. if ( name.length > 250 ) name = name.substring(0, 250) + '\u2026';
  446. var value = htmlToDiscord(sectionContent, embed.url).trim().replace( /\n{3,}/g, '\n\n' );
  447. if ( value.length > 1000 ) value = limitLength(value, 1000, 20);
  448. if ( name.length && value.length ) {
  449. embed.spliceFields( 0, 0, {name, value} );
  450. if ( newFragment ) {
  451. embed.setURL( pagelink.replace( toSection(fragment), toSection(newFragment) ) );
  452. content = content.replace( '<' + pagelink + '>', '<' + embed.url + '>' );
  453. }
  454. }
  455. else if ( embed.backupField ) {
  456. embed.spliceFields( 0, 0, embed.backupField );
  457. }
  458. }
  459. else if ( embed.backupField ) {
  460. embed.spliceFields( 0, 0, embed.backupField );
  461. }
  462. }
  463. if ( !embed.description && embed.length < 5000 ) {
  464. if ( contentmodel !== 'wikitext' || disambiguation === undefined || fragment ) {
  465. $('h1, h2, h3, h4, h5, h6').nextAll().remove();
  466. $('h1, h2, h3, h4, h5, h6').remove();
  467. }
  468. $(infoboxList.join(', ')).remove();
  469. $('div, ' + removeClasses.join(', '), $('.mw-parser-output')).not(removeClassesExceptions.join(', ')).remove();
  470. var description = htmlToDiscord($.html(), embed.url, true).trim().replace( /\n{3,}/g, '\n\n' );
  471. if ( description ) {
  472. if ( disambiguation !== undefined && !fragment && embed.length < 4250 ) {
  473. if ( description.length > 1500 ) description = limitLength(description, 1500, 250);
  474. }
  475. else if ( fragment && description.length > 500 ) description = limitLength(description, 500, 250);
  476. else if ( description.length > 1000 ) description = limitLength(description, 1000, 500);
  477. embed.setDescription( description );
  478. }
  479. else if ( embed.backupDescription ) {
  480. embed.setDescription( embed.backupDescription );
  481. }
  482. }
  483. }, error => {
  484. console.log( '- Error while parsing the page: ' + error );
  485. if ( embed.backupDescription && embed.length < 5000 ) {
  486. embed.setDescription( embed.backupDescription );
  487. }
  488. if ( embed.backupField && embed.length < 4750 && embed.fields.length < 25 ) {
  489. embed.spliceFields( 0, 0, embed.backupField );
  490. }
  491. } ).then( () => {
  492. let embeds = [embed];
  493. if ( extraImages.length ) {
  494. if ( !embed.image ) embed.setImage( extraImages.shift() );
  495. extraImages.slice(0, 10).forEach( extraImage => {
  496. let imageEmbed = new MessageEmbed().setURL( embed.url ).setImage( extraImage );
  497. if ( embeds.length < 5 && embeds.reduce( (acc, val) => acc + val.length, imageEmbed.length ) <= 5500 ) embeds.push(imageEmbed);
  498. } );
  499. }
  500. return message.edit( {content, embeds} ).catch(log_error);
  501. } );
  502. } );
  503. }