Markdown Versions of Posts for AI Crawlers, Plus llms.txt
Serves any post or page as clean Markdown when .md is added to its address (or ?format=markdown), and publishes an /llms.txt index of your pages and recent posts for AI tools.
Opens WPIDE on
Code
// Post types that get a Markdown version.
$wpide_md_post_types = array( 'post', 'page' );
// How many recent posts and top-level pages /llms.txt lists (0 leaves the section out).
$wpide_llms_posts = 30;
$wpide_llms_pages = 20;
// Keep the Markdown copies out of search results. The normal page stays the original.
$wpide_md_noindex = true;
$wpide_md_plain = function ( $text ) {
$text = html_entity_decode( wp_strip_all_tags( (string) $text ), ENT_QUOTES | ENT_HTML5, 'UTF-8' );
return trim( (string) preg_replace( '/\s+/u', ' ', $text ) );
};
$wpide_md_target = function ( $url ) {
$url = trim( (string) $url );
if ( '' !== $url && ! preg_match( '#^[a-z][a-z0-9+.-]*:#i', $url ) ) {
$url = WP_Http::make_absolute_url( $url, home_url( '/' ) );
}
return str_replace( array( ' ', '(', ')' ), array( '%20', '%28', '%29' ), $url );
};
// Markdown address of a post: /post-name.md, or ?format=markdown for plain permalinks and the front page.
$wpide_md_url = function ( $post ) {
$permalink = get_permalink( $post );
if ( ! $permalink ) {
return '';
}
if ( false !== strpos( $permalink, '?' ) || untrailingslashit( $permalink ) === untrailingslashit( home_url() ) ) {
return add_query_arg( 'format', 'markdown', $permalink );
}
return untrailingslashit( $permalink ) . '.md';
};
// Small HTML to Markdown converter: headings, paragraphs, emphasis, links, images, lists, quotes, code and tables.
$wpide_md_convert = function ( DOMNode $parent ) use ( &$wpide_md_convert, $wpide_md_plain, $wpide_md_target ) {
$out = '';
$inline = function ( $raw, $marker ) {
$inner = trim( $raw );
if ( '' === $inner ) {
return '' === $raw ? '' : ' ';
}
return ( preg_match( '/^\s/u', $raw ) ? ' ' : '' ) . $marker . $inner . $marker . ( preg_match( '/\s$/u', $raw ) ? ' ' : '' );
};
foreach ( $parent->childNodes as $node ) {
if ( XML_TEXT_NODE === $node->nodeType || XML_CDATA_SECTION_NODE === $node->nodeType ) {
$out .= (string) preg_replace( '/\s+/u', ' ', $node->nodeValue );
continue;
}
if ( XML_ELEMENT_NODE !== $node->nodeType ) {
continue;
}
$tag = strtolower( $node->nodeName );
switch ( $tag ) {
case 'script':
case 'style':
case 'noscript':
case 'template':
case 'svg':
case 'form':
case 'button':
case 'input':
case 'select':
case 'textarea':
break;
case 'h1':
case 'h2':
case 'h3':
case 'h4':
case 'h5':
case 'h6':
$text = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
if ( '' !== $text ) {
$out .= "\n\n" . str_repeat( '#', (int) substr( $tag, 1 ) ) . ' ' . $text . "\n\n";
}
break;
case 'p':
$out .= "\n\n" . trim( $wpide_md_convert( $node ) ) . "\n\n";
break;
case 'br':
$out .= " \n";
break;
case 'hr':
$out .= "\n\n---\n\n";
break;
case 'strong':
case 'b':
$out .= $inline( $wpide_md_convert( $node ), '**' );
break;
case 'em':
case 'i':
$out .= $inline( $wpide_md_convert( $node ), '*' );
break;
case 'del':
case 's':
case 'strike':
$out .= $inline( $wpide_md_convert( $node ), '~~' );
break;
case 'code':
$code = $node->textContent;
if ( '' !== trim( $code ) ) {
$out .= false !== strpos( $code, '`' ) ? '`` ' . $code . ' ``' : '`' . $code . '`';
}
break;
case 'pre':
$language = '';
$classes = $node->getAttribute( 'class' );
foreach ( $node->getElementsByTagName( 'code' ) as $code_node ) {
$classes .= ' ' . $code_node->getAttribute( 'class' );
break;
}
if ( preg_match( '/(?:language|lang)-([\w+#.-]+)/', $classes, $match ) ) {
$language = $match[1];
}
$code = rtrim( str_replace( "\r\n", "\n", $node->textContent ) );
$fence = false !== strpos( $code, '```' ) ? '~~~' : '```';
$out .= "\n\n" . $fence . $language . "\n" . $code . "\n" . $fence . "\n\n";
break;
case 'blockquote':
$quote = trim( (string) preg_replace( "/\n{3,}/", "\n\n", $wpide_md_convert( $node ) ) );
if ( '' !== $quote ) {
$out .= "\n\n" . preg_replace( '/^/m', '> ', $quote ) . "\n\n";
}
break;
case 'ul':
case 'ol':
$number = 'ol' === $tag ? max( 1, (int) $node->getAttribute( 'start' ) ) : 0;
$items = array();
foreach ( $node->childNodes as $item ) {
if ( XML_ELEMENT_NODE !== $item->nodeType || 'li' !== strtolower( $item->nodeName ) ) {
continue;
}
$text = trim( (string) preg_replace( "/\n\s*\n/", "\n", $wpide_md_convert( $item ) ) );
if ( '' === $text ) {
continue;
}
$marker = $number ? ( $number++ ) . '. ' : '- ';
$items[] = $marker . str_replace( "\n", "\n" . str_repeat( ' ', strlen( $marker ) ), $text );
}
if ( $items ) {
$out .= "\n\n" . implode( "\n", $items ) . "\n\n";
}
break;
case 'li':
$out .= "\n- " . trim( $wpide_md_convert( $node ) ) . "\n";
break;
case 'a':
$href = trim( $node->getAttribute( 'href' ) );
$raw = $wpide_md_convert( $node );
$label = trim( (string) preg_replace( '/\s+/u', ' ', $raw ) );
if ( '' === $label ) {
break;
}
if ( '' === $href || '#' === $href[0] || 0 === stripos( $href, 'javascript:' ) ) {
$out .= $raw;
break;
}
$out .= ( preg_match( '/^\s/u', $raw ) ? ' ' : '' ) . '[' . $label . '](' . $wpide_md_target( $href ) . ')' . ( preg_match( '/\s$/u', $raw ) ? ' ' : '' );
break;
case 'img':
$src = $node->getAttribute( 'src' );
foreach ( array( 'data-src', 'data-lazy-src', 'data-original' ) as $attribute ) {
if ( ( '' === $src || 0 === strpos( $src, 'data:' ) ) && $node->getAttribute( $attribute ) ) {
$src = $node->getAttribute( $attribute );
}
}
if ( '' !== $src && 0 !== strpos( $src, 'data:' ) ) {
$alt = str_replace( array( '[', ']' ), '', $wpide_md_plain( $node->getAttribute( 'alt' ) ) );
$out .= ' . ')';
}
break;
case 'iframe':
case 'video':
case 'audio':
case 'embed':
case 'object':
$src = $node->getAttribute( 'src' ) ? $node->getAttribute( 'src' ) : $node->getAttribute( 'data' );
if ( ! $src ) {
foreach ( $node->getElementsByTagName( 'source' ) as $source ) {
$src = $source->getAttribute( 'src' );
if ( $src ) {
break;
}
}
}
if ( $src && 0 !== strpos( $src, 'data:' ) ) {
$title = $wpide_md_plain( $node->getAttribute( 'title' ) );
$title = '' !== $title ? $title : ( 'iframe' === $tag ? 'Embedded content' : ucfirst( $tag ) );
$out .= "\n\n[" . $title . '](' . $wpide_md_target( $src ) . ")\n\n";
}
break;
case 'figcaption':
$caption = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
if ( '' !== $caption ) {
$out .= "\n\n*" . $caption . "*\n\n";
}
break;
case 'summary':
case 'dt':
$term = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
if ( '' !== $term ) {
$out .= "\n\n**" . $term . "**\n\n";
}
break;
case 'table':
$rows = array();
foreach ( $node->getElementsByTagName( 'tr' ) as $row ) {
$cells = array();
foreach ( $row->childNodes as $cell ) {
if ( XML_ELEMENT_NODE === $cell->nodeType && in_array( strtolower( $cell->nodeName ), array( 'td', 'th' ), true ) ) {
$cells[] = str_replace( '|', '\|', trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $cell ) ) ) );
}
}
if ( $cells ) {
$rows[] = $cells;
}
}
if ( $rows ) {
$columns = max( array_map( 'count', $rows ) );
$lines = array();
foreach ( $rows as $index => $cells ) {
$lines[] = '| ' . implode( ' | ', array_pad( $cells, $columns, '' ) ) . ' |';
if ( 0 === $index ) {
$lines[] = '|' . str_repeat( ' --- |', $columns );
}
}
$out .= "\n\n" . implode( "\n", $lines ) . "\n\n";
}
break;
case 'div':
case 'section':
case 'article':
case 'aside':
case 'header':
case 'footer':
case 'main':
case 'nav':
case 'figure':
case 'details':
case 'dl':
case 'dd':
case 'address':
$out .= "\n\n" . $wpide_md_convert( $node ) . "\n\n";
break;
default:
$out .= $wpide_md_convert( $node );
}
}
return $out;
};
$wpide_md_from_html = function ( $html ) use ( $wpide_md_convert ) {
if ( ! class_exists( 'DOMDocument' ) ) {
return trim( wp_strip_all_tags( $html ) );
}
$document = new DOMDocument();
$previous = libxml_use_internal_errors( true );
$document->loadHTML( '<?xml encoding="utf-8"?><div>' . $html . '</div>', LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
libxml_clear_errors();
libxml_use_internal_errors( $previous );
$root = $document->getElementsByTagName( 'div' )->item( 0 );
if ( ! $root ) {
return '';
}
$markdown = preg_replace( '/^[ \t]+$/m', '', $wpide_md_convert( $root ) );
return trim( (string) preg_replace( "/\n{3,}/", "\n\n", (string) $markdown ) );
};
$wpide_md_send = function ( $post ) use ( $wpide_md_post_types, $wpide_md_noindex, $wpide_md_plain, $wpide_md_from_html ) {
if ( ! $post instanceof WP_Post || ! in_array( $post->post_type, $wpide_md_post_types, true ) || ! is_post_publicly_viewable( $post ) || post_password_required( $post ) ) {
return;
}
$GLOBALS['post'] = $post;
setup_postdata( $post );
$html = apply_filters( 'the_content', $post->post_content );
wp_reset_postdata();
$json = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES;
$title = $wpide_md_plain( get_the_title( $post ) );
$meta = array(
'---',
'title: ' . wp_json_encode( $title, $json ),
'url: ' . get_permalink( $post ),
'date: ' . get_the_date( 'Y-m-d', $post ),
'modified: ' . get_the_modified_date( 'Y-m-d', $post ),
);
if ( post_type_supports( $post->post_type, 'author' ) ) {
$meta[] = 'author: ' . wp_json_encode( $wpide_md_plain( get_the_author_meta( 'display_name', $post->post_author ) ), $json );
}
if ( has_excerpt( $post ) ) {
$meta[] = 'description: ' . wp_json_encode( $wpide_md_plain( $post->post_excerpt ), $json );
}
$meta[] = '---';
status_header( 200 );
header( 'Content-Type: text/markdown; charset=utf-8' );
header( 'X-Content-Type-Options: nosniff' );
header( 'Link: <' . esc_url_raw( get_permalink( $post ) ) . '>; rel="canonical"' );
if ( $wpide_md_noindex ) {
header( 'X-Robots-Tag: noindex' );
}
echo implode( "\n", $meta ) . "\n\n# " . $title . "\n\n" . $wpide_md_from_html( $html ) . "\n"; // phpcs:ignore WordPress.Security.EscapeOutput -- Markdown, not HTML.
exit;
};
$wpide_llms_send = function () use ( $wpide_md_post_types, $wpide_llms_posts, $wpide_llms_pages, $wpide_md_plain, $wpide_md_url ) {
$item = function ( $post ) use ( $wpide_md_plain, $wpide_md_url ) {
$title = str_replace( array( '[', ']' ), array( '(', ')' ), $wpide_md_plain( get_the_title( $post ) ) );
$description = $wpide_md_plain( has_excerpt( $post ) ? $post->post_excerpt : wp_trim_words( strip_shortcodes( $post->post_content ), 25, '...' ) );
return '- [' . $title . '](' . $wpide_md_url( $post ) . ')' . ( '' !== $description ? ': ' . $description : '' );
};
$out = '# ' . $wpide_md_plain( get_bloginfo( 'name' ) ) . "\n\n";
$tagline = $wpide_md_plain( get_bloginfo( 'description' ) );
if ( '' !== $tagline ) {
$out .= '> ' . $tagline . "\n\n";
}
$out .= "The links below point to Markdown versions of each page.\n";
if ( $wpide_llms_pages > 0 && in_array( 'page', $wpide_md_post_types, true ) ) {
$pages = get_posts( array(
'post_type' => 'page',
'post_parent' => 0,
'numberposts' => (int) $wpide_llms_pages,
'orderby' => array( 'menu_order' => 'ASC', 'title' => 'ASC' ),
'has_password' => false,
) );
if ( $pages ) {
$out .= "\n## Pages\n\n" . implode( "\n", array_map( $item, $pages ) ) . "\n";
}
}
$other_types = array_values( array_diff( $wpide_md_post_types, array( 'page' ) ) );
if ( $wpide_llms_posts > 0 && $other_types ) {
$posts = get_posts( array(
'post_type' => $other_types,
'numberposts' => (int) $wpide_llms_posts,
'has_password' => false,
) );
if ( $posts ) {
$out .= "\n## Recent posts\n\n" . implode( "\n", array_map( $item, $posts ) ) . "\n";
}
}
status_header( 200 );
header( 'Content-Type: text/plain; charset=utf-8' );
header( 'X-Content-Type-Options: nosniff' );
echo $out; // phpcs:ignore WordPress.Security.EscapeOutput -- Plain text.
exit;
};
// /llms.txt and /any-post-address.md are answered before WordPress looks for a matching page.
add_action( 'parse_request', function ( $wp ) use ( $wpide_md_send, $wpide_llms_send ) {
$request = isset( $wp->request ) ? (string) $wp->request : '';
if ( 'llms.txt' === $request ) {
$wpide_llms_send();
}
if ( strlen( $request ) > 3 && '.md' === substr( $request, -3 ) ) {
$path = substr( $request, 0, -3 );
$post_id = url_to_postid( home_url( user_trailingslashit( $path ) ) );
if ( ! $post_id ) {
$post_id = url_to_postid( home_url( '/' . $path ) );
}
if ( $post_id ) {
$wpide_md_send( get_post( $post_id ) );
}
}
} );
// Any post address with ?format=markdown.
add_action( 'template_redirect', function () use ( $wpide_md_post_types, $wpide_md_send ) {
if ( isset( $_GET['format'] ) && 'markdown' === sanitize_key( wp_unslash( $_GET['format'] ) ) && is_singular( $wpide_md_post_types ) ) { // phpcs:ignore WordPress.Security.NonceVerification
$wpide_md_send( get_queried_object() );
}
}, 1 );
// Point crawlers to the Markdown version from the normal page.
add_action( 'wp_head', function () use ( $wpide_md_post_types, $wpide_md_url ) {
$post = get_queried_object();
if ( ! is_singular( $wpide_md_post_types ) || ! $post instanceof WP_Post || ! is_post_publicly_viewable( $post ) || post_password_required( $post ) ) {
return;
}
$url = $wpide_md_url( $post );
if ( $url ) {
echo '<link rel="alternate" type="text/markdown" href="' . esc_url( $url ) . '">' . "\n";
}
} );Notes
Try it: open a post and add .md to the end of its address (example.com/my-post.md). With plain permalinks, and for the front page, use ?format=markdown instead. Each page also gets a <link rel="alternate" type="text/markdown"> tag. If a real llms.txt file exists in the site root, the server shows that file instead. If a caching plugin serves the .md addresses as HTML, exclude them from the cache. Uses PHP's DOM extension (standard on almost every host); without it the text is sent unformatted.
How to add this snippet
- Install WPIDE on your WordPress site (it is free) and open Code Snippets.
- Use snippet above adds it from the library in one click. Or copy the code into a new snippet with the same code type.
- It is added inactive: review the code and settings, then switch it on.