# Markdown Versions of Posts for AI Crawlers, Plus llms.txt

> Serves any post or page as clean Markdown when .md is added to its address (or ?format=markdown), and publishes an /llms.txt index of your pages and recent posts for AI tools.

- Type: PHP
- Runs: Frontend only
- Category: [SEO](https://wpide.com/library/category/seo/)
- Tags: seo, ai, markdown, llms-txt
- Author: WPIDE Team
- Installs: 0
- Updated: 2026-10-28
- Page: https://wpide.com/library/markdown-versions-for-ai-crawlers/

## Code

````php
// Post types that get a Markdown version.
$wpide_md_post_types = array( 'post', 'page' );

// How many recent posts and top-level pages /llms.txt lists (0 leaves the section out).
$wpide_llms_posts = 30;
$wpide_llms_pages = 20;

// Keep the Markdown copies out of search results. The normal page stays the original.
$wpide_md_noindex = true;

$wpide_md_plain = function ( $text ) {
    $text = html_entity_decode( wp_strip_all_tags( (string) $text ), ENT_QUOTES | ENT_HTML5, 'UTF-8' );

    return trim( (string) preg_replace( '/\s+/u', ' ', $text ) );
};

$wpide_md_target = function ( $url ) {
    $url = trim( (string) $url );

    if ( '' !== $url && ! preg_match( '#^[a-z][a-z0-9+.-]*:#i', $url ) ) {
        $url = WP_Http::make_absolute_url( $url, home_url( '/' ) );
    }

    return str_replace( array( ' ', '(', ')' ), array( '%20', '%28', '%29' ), $url );
};

// Markdown address of a post: /post-name.md, or ?format=markdown for plain permalinks and the front page.
$wpide_md_url = function ( $post ) {
    $permalink = get_permalink( $post );

    if ( ! $permalink ) {
        return '';
    }

    if ( false !== strpos( $permalink, '?' ) || untrailingslashit( $permalink ) === untrailingslashit( home_url() ) ) {
        return add_query_arg( 'format', 'markdown', $permalink );
    }

    return untrailingslashit( $permalink ) . '.md';
};

// Small HTML to Markdown converter: headings, paragraphs, emphasis, links, images, lists, quotes, code and tables.
$wpide_md_convert = function ( DOMNode $parent ) use ( &$wpide_md_convert, $wpide_md_plain, $wpide_md_target ) {
    $out = '';

    $inline = function ( $raw, $marker ) {
        $inner = trim( $raw );

        if ( '' === $inner ) {
            return '' === $raw ? '' : ' ';
        }

        return ( preg_match( '/^\s/u', $raw ) ? ' ' : '' ) . $marker . $inner . $marker . ( preg_match( '/\s$/u', $raw ) ? ' ' : '' );
    };

    foreach ( $parent->childNodes as $node ) {
        if ( XML_TEXT_NODE === $node->nodeType || XML_CDATA_SECTION_NODE === $node->nodeType ) {
            $out .= (string) preg_replace( '/\s+/u', ' ', $node->nodeValue );
            continue;
        }

        if ( XML_ELEMENT_NODE !== $node->nodeType ) {
            continue;
        }

        $tag = strtolower( $node->nodeName );

        switch ( $tag ) {
            case 'script':
            case 'style':
            case 'noscript':
            case 'template':
            case 'svg':
            case 'form':
            case 'button':
            case 'input':
            case 'select':
            case 'textarea':
                break;

            case 'h1':
            case 'h2':
            case 'h3':
            case 'h4':
            case 'h5':
            case 'h6':
                $text = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
                if ( '' !== $text ) {
                    $out .= "\n\n" . str_repeat( '#', (int) substr( $tag, 1 ) ) . ' ' . $text . "\n\n";
                }
                break;

            case 'p':
                $out .= "\n\n" . trim( $wpide_md_convert( $node ) ) . "\n\n";
                break;

            case 'br':
                $out .= "  \n";
                break;

            case 'hr':
                $out .= "\n\n---\n\n";
                break;

            case 'strong':
            case 'b':
                $out .= $inline( $wpide_md_convert( $node ), '**' );
                break;

            case 'em':
            case 'i':
                $out .= $inline( $wpide_md_convert( $node ), '*' );
                break;

            case 'del':
            case 's':
            case 'strike':
                $out .= $inline( $wpide_md_convert( $node ), '~~' );
                break;

            case 'code':
                $code = $node->textContent;
                if ( '' !== trim( $code ) ) {
                    $out .= false !== strpos( $code, '`' ) ? '`` ' . $code . ' ``' : '`' . $code . '`';
                }
                break;

            case 'pre':
                $language = '';
                $classes  = $node->getAttribute( 'class' );
                foreach ( $node->getElementsByTagName( 'code' ) as $code_node ) {
                    $classes .= ' ' . $code_node->getAttribute( 'class' );
                    break;
                }
                if ( preg_match( '/(?:language|lang)-([\w+#.-]+)/', $classes, $match ) ) {
                    $language = $match[1];
                }
                $code  = rtrim( str_replace( "\r\n", "\n", $node->textContent ) );
                $fence = false !== strpos( $code, '```' ) ? '~~~' : '```';
                $out  .= "\n\n" . $fence . $language . "\n" . $code . "\n" . $fence . "\n\n";
                break;

            case 'blockquote':
                $quote = trim( (string) preg_replace( "/\n{3,}/", "\n\n", $wpide_md_convert( $node ) ) );
                if ( '' !== $quote ) {
                    $out .= "\n\n" . preg_replace( '/^/m', '> ', $quote ) . "\n\n";
                }
                break;

            case 'ul':
            case 'ol':
                $number = 'ol' === $tag ? max( 1, (int) $node->getAttribute( 'start' ) ) : 0;
                $items  = array();
                foreach ( $node->childNodes as $item ) {
                    if ( XML_ELEMENT_NODE !== $item->nodeType || 'li' !== strtolower( $item->nodeName ) ) {
                        continue;
                    }
                    $text = trim( (string) preg_replace( "/\n\s*\n/", "\n", $wpide_md_convert( $item ) ) );
                    if ( '' === $text ) {
                        continue;
                    }
                    $marker  = $number ? ( $number++ ) . '. ' : '- ';
                    $items[] = $marker . str_replace( "\n", "\n" . str_repeat( ' ', strlen( $marker ) ), $text );
                }
                if ( $items ) {
                    $out .= "\n\n" . implode( "\n", $items ) . "\n\n";
                }
                break;

            case 'li':
                $out .= "\n- " . trim( $wpide_md_convert( $node ) ) . "\n";
                break;

            case 'a':
                $href  = trim( $node->getAttribute( 'href' ) );
                $raw   = $wpide_md_convert( $node );
                $label = trim( (string) preg_replace( '/\s+/u', ' ', $raw ) );
                if ( '' === $label ) {
                    break;
                }
                if ( '' === $href || '#' === $href[0] || 0 === stripos( $href, 'javascript:' ) ) {
                    $out .= $raw;
                    break;
                }
                $out .= ( preg_match( '/^\s/u', $raw ) ? ' ' : '' ) . '[' . $label . '](' . $wpide_md_target( $href ) . ')' . ( preg_match( '/\s$/u', $raw ) ? ' ' : '' );
                break;

            case 'img':
                $src = $node->getAttribute( 'src' );
                foreach ( array( 'data-src', 'data-lazy-src', 'data-original' ) as $attribute ) {
                    if ( ( '' === $src || 0 === strpos( $src, 'data:' ) ) && $node->getAttribute( $attribute ) ) {
                        $src = $node->getAttribute( $attribute );
                    }
                }
                if ( '' !== $src && 0 !== strpos( $src, 'data:' ) ) {
                    $alt  = str_replace( array( '[', ']' ), '', $wpide_md_plain( $node->getAttribute( 'alt' ) ) );
                    $out .= '![' . $alt . '](' . $wpide_md_target( $src ) . ')';
                }
                break;

            case 'iframe':
            case 'video':
            case 'audio':
            case 'embed':
            case 'object':
                $src = $node->getAttribute( 'src' ) ? $node->getAttribute( 'src' ) : $node->getAttribute( 'data' );
                if ( ! $src ) {
                    foreach ( $node->getElementsByTagName( 'source' ) as $source ) {
                        $src = $source->getAttribute( 'src' );
                        if ( $src ) {
                            break;
                        }
                    }
                }
                if ( $src && 0 !== strpos( $src, 'data:' ) ) {
                    $title = $wpide_md_plain( $node->getAttribute( 'title' ) );
                    $title = '' !== $title ? $title : ( 'iframe' === $tag ? 'Embedded content' : ucfirst( $tag ) );
                    $out  .= "\n\n[" . $title . '](' . $wpide_md_target( $src ) . ")\n\n";
                }
                break;

            case 'figcaption':
                $caption = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
                if ( '' !== $caption ) {
                    $out .= "\n\n*" . $caption . "*\n\n";
                }
                break;

            case 'summary':
            case 'dt':
                $term = trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $node ) ) );
                if ( '' !== $term ) {
                    $out .= "\n\n**" . $term . "**\n\n";
                }
                break;

            case 'table':
                $rows = array();
                foreach ( $node->getElementsByTagName( 'tr' ) as $row ) {
                    $cells = array();
                    foreach ( $row->childNodes as $cell ) {
                        if ( XML_ELEMENT_NODE === $cell->nodeType && in_array( strtolower( $cell->nodeName ), array( 'td', 'th' ), true ) ) {
                            $cells[] = str_replace( '|', '\|', trim( (string) preg_replace( '/\s+/u', ' ', $wpide_md_convert( $cell ) ) ) );
                        }
                    }
                    if ( $cells ) {
                        $rows[] = $cells;
                    }
                }
                if ( $rows ) {
                    $columns = max( array_map( 'count', $rows ) );
                    $lines   = array();
                    foreach ( $rows as $index => $cells ) {
                        $lines[] = '| ' . implode( ' | ', array_pad( $cells, $columns, '' ) ) . ' |';
                        if ( 0 === $index ) {
                            $lines[] = '|' . str_repeat( ' --- |', $columns );
                        }
                    }
                    $out .= "\n\n" . implode( "\n", $lines ) . "\n\n";
                }
                break;

            case 'div':
            case 'section':
            case 'article':
            case 'aside':
            case 'header':
            case 'footer':
            case 'main':
            case 'nav':
            case 'figure':
            case 'details':
            case 'dl':
            case 'dd':
            case 'address':
                $out .= "\n\n" . $wpide_md_convert( $node ) . "\n\n";
                break;

            default:
                $out .= $wpide_md_convert( $node );
        }
    }

    return $out;
};

$wpide_md_from_html = function ( $html ) use ( $wpide_md_convert ) {
    if ( ! class_exists( 'DOMDocument' ) ) {
        return trim( wp_strip_all_tags( $html ) );
    }

    $document = new DOMDocument();
    $previous = libxml_use_internal_errors( true );
    $document->loadHTML( '<?xml encoding="utf-8"?><div>' . $html . '</div>', LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD );
    libxml_clear_errors();
    libxml_use_internal_errors( $previous );

    $root = $document->getElementsByTagName( 'div' )->item( 0 );

    if ( ! $root ) {
        return '';
    }

    $markdown = preg_replace( '/^[ \t]+$/m', '', $wpide_md_convert( $root ) );

    return trim( (string) preg_replace( "/\n{3,}/", "\n\n", (string) $markdown ) );
};

$wpide_md_send = function ( $post ) use ( $wpide_md_post_types, $wpide_md_noindex, $wpide_md_plain, $wpide_md_from_html ) {
    if ( ! $post instanceof WP_Post || ! in_array( $post->post_type, $wpide_md_post_types, true ) || ! is_post_publicly_viewable( $post ) || post_password_required( $post ) ) {
        return;
    }

    $GLOBALS['post'] = $post;
    setup_postdata( $post );
    $html = apply_filters( 'the_content', $post->post_content );
    wp_reset_postdata();

    $json  = JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES;
    $title = $wpide_md_plain( get_the_title( $post ) );
    $meta  = array(
        '---',
        'title: ' . wp_json_encode( $title, $json ),
        'url: ' . get_permalink( $post ),
        'date: ' . get_the_date( 'Y-m-d', $post ),
        'modified: ' . get_the_modified_date( 'Y-m-d', $post ),
    );

    if ( post_type_supports( $post->post_type, 'author' ) ) {
        $meta[] = 'author: ' . wp_json_encode( $wpide_md_plain( get_the_author_meta( 'display_name', $post->post_author ) ), $json );
    }

    if ( has_excerpt( $post ) ) {
        $meta[] = 'description: ' . wp_json_encode( $wpide_md_plain( $post->post_excerpt ), $json );
    }

    $meta[] = '---';

    status_header( 200 );
    header( 'Content-Type: text/markdown; charset=utf-8' );
    header( 'X-Content-Type-Options: nosniff' );
    header( 'Link: <' . esc_url_raw( get_permalink( $post ) ) . '>; rel="canonical"' );

    if ( $wpide_md_noindex ) {
        header( 'X-Robots-Tag: noindex' );
    }

    echo implode( "\n", $meta ) . "\n\n# " . $title . "\n\n" . $wpide_md_from_html( $html ) . "\n"; // phpcs:ignore WordPress.Security.EscapeOutput -- Markdown, not HTML.
    exit;
};

$wpide_llms_send = function () use ( $wpide_md_post_types, $wpide_llms_posts, $wpide_llms_pages, $wpide_md_plain, $wpide_md_url ) {
    $item = function ( $post ) use ( $wpide_md_plain, $wpide_md_url ) {
        $title       = str_replace( array( '[', ']' ), array( '(', ')' ), $wpide_md_plain( get_the_title( $post ) ) );
        $description = $wpide_md_plain( has_excerpt( $post ) ? $post->post_excerpt : wp_trim_words( strip_shortcodes( $post->post_content ), 25, '...' ) );

        return '- [' . $title . '](' . $wpide_md_url( $post ) . ')' . ( '' !== $description ? ': ' . $description : '' );
    };

    $out     = '# ' . $wpide_md_plain( get_bloginfo( 'name' ) ) . "\n\n";
    $tagline = $wpide_md_plain( get_bloginfo( 'description' ) );

    if ( '' !== $tagline ) {
        $out .= '> ' . $tagline . "\n\n";
    }

    $out .= "The links below point to Markdown versions of each page.\n";

    if ( $wpide_llms_pages > 0 && in_array( 'page', $wpide_md_post_types, true ) ) {
        $pages = get_posts( array(
            'post_type'    => 'page',
            'post_parent'  => 0,
            'numberposts'  => (int) $wpide_llms_pages,
            'orderby'      => array( 'menu_order' => 'ASC', 'title' => 'ASC' ),
            'has_password' => false,
        ) );

        if ( $pages ) {
            $out .= "\n## Pages\n\n" . implode( "\n", array_map( $item, $pages ) ) . "\n";
        }
    }

    $other_types = array_values( array_diff( $wpide_md_post_types, array( 'page' ) ) );

    if ( $wpide_llms_posts > 0 && $other_types ) {
        $posts = get_posts( array(
            'post_type'    => $other_types,
            'numberposts'  => (int) $wpide_llms_posts,
            'has_password' => false,
        ) );

        if ( $posts ) {
            $out .= "\n## Recent posts\n\n" . implode( "\n", array_map( $item, $posts ) ) . "\n";
        }
    }

    status_header( 200 );
    header( 'Content-Type: text/plain; charset=utf-8' );
    header( 'X-Content-Type-Options: nosniff' );

    echo $out; // phpcs:ignore WordPress.Security.EscapeOutput -- Plain text.
    exit;
};

// /llms.txt and /any-post-address.md are answered before WordPress looks for a matching page.
add_action( 'parse_request', function ( $wp ) use ( $wpide_md_send, $wpide_llms_send ) {
    $request = isset( $wp->request ) ? (string) $wp->request : '';

    if ( 'llms.txt' === $request ) {
        $wpide_llms_send();
    }

    if ( strlen( $request ) > 3 && '.md' === substr( $request, -3 ) ) {
        $path    = substr( $request, 0, -3 );
        $post_id = url_to_postid( home_url( user_trailingslashit( $path ) ) );

        if ( ! $post_id ) {
            $post_id = url_to_postid( home_url( '/' . $path ) );
        }

        if ( $post_id ) {
            $wpide_md_send( get_post( $post_id ) );
        }
    }
} );

// Any post address with ?format=markdown.
add_action( 'template_redirect', function () use ( $wpide_md_post_types, $wpide_md_send ) {
    if ( isset( $_GET['format'] ) && 'markdown' === sanitize_key( wp_unslash( $_GET['format'] ) ) && is_singular( $wpide_md_post_types ) ) { // phpcs:ignore WordPress.Security.NonceVerification
        $wpide_md_send( get_queried_object() );
    }
}, 1 );

// Point crawlers to the Markdown version from the normal page.
add_action( 'wp_head', function () use ( $wpide_md_post_types, $wpide_md_url ) {
    $post = get_queried_object();

    if ( ! is_singular( $wpide_md_post_types ) || ! $post instanceof WP_Post || ! is_post_publicly_viewable( $post ) || post_password_required( $post ) ) {
        return;
    }

    $url = $wpide_md_url( $post );

    if ( $url ) {
        echo '<link rel="alternate" type="text/markdown" href="' . esc_url( $url ) . '">' . "\n";
    }
} );
````

## Notes

Try it: open a post and add .md to the end of its address (example.com/my-post.md). With plain permalinks, and for the front page, use ?format=markdown instead. Each page also gets a <link rel="alternate" type="text/markdown"> tag. If a real llms.txt file exists in the site root, the server shows that file instead. If a caching plugin serves the .md addresses as HTML, exclude them from the cache. Uses PHP's DOM extension (standard on almost every host); without it the text is sent unformatted.

## How to use it

Install the free WPIDE plugin (https://wordpress.org/plugins/wpide/), find "Markdown Versions of Posts for AI Crawlers, Plus llms.txt" in its snippet library and add it in one click; or create a new PHP snippet in WPIDE (or any code snippets plugin) and paste the code above. More snippets: https://wpide.com/library/