From 92acb5344effca88d7a334f0e77e0fe62edbef0a Mon Sep 17 00:00:00 2001 From: "Collin J. Doering" Date: Wed, 5 Aug 2015 02:42:12 -0400 Subject: [PATCH] Add robots.txt for web robots Allow all robots to access any part of the site. This may change in the future, as the html snippets that are generated for the posts and pages should not be directly indexed. See: http://www.robotstxt.org/robotstxt.html Signed-off-by: Collin J. Doering --- robots.txt | 2 ++ src/site.hs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 robots.txt diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..c2a49f4 --- /dev/null +++ b/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Allow: / diff --git a/src/site.hs b/src/site.hs index 06485bb..f78841a 100644 --- a/src/site.hs +++ b/src/site.hs @@ -105,7 +105,7 @@ main = do hakyllWith myConfig $ do -- All Versions ------------------------------------------------------------------------------------------ - match ("action/**" .||. "files/**" .||. "images/**" .||. "fonts/**") $ do + match ("action/**" .||. "files/**" .||. "images/**" .||. "fonts/**" .||. "robots.txt") $ do route idRoute compile copyFileCompiler