1
using System;
2
using System.Collections.Generic;
3
using System.Linq;
4
using System.Text;
5
using TurnerSoftware.RobotsExclusionTools.Helpers;
6

7
namespace TurnerSoftware.RobotsExclusionTools
8
{
9
	public class RobotsFile
10
	{
11 1
		public Uri BaseUri { get; }
12

13 1
		public IEnumerable<SiteAccessEntry> SiteAccessEntries { get; set; } = Enumerable.Empty<SiteAccessEntry>();
14 1
		public IEnumerable<SitemapUrlEntry> SitemapEntries { get; set; } = Enumerable.Empty<SitemapUrlEntry>();
15

16 1
		public RobotsFile(Uri baseUri)
17 1
		{
18 1
			BaseUri = baseUri;
19 1
		}
20

21
		public static RobotsFile AllowAllRobots(Uri baseUri)
22 1
		{
23 1
			return new RobotsFile(baseUri);
24 1
		}
25

26
		public static RobotsFile DenyAllRobots(Uri baseUri)
27 1
		{
28 1
			return new RobotsFile(baseUri)
29 1
			{
30 1
				SiteAccessEntries = new []
31 1
				{
32 1
					new SiteAccessEntry
33 1
					{
34 1
						UserAgents = new[] { "*" },
35 1
						PathRules = new []
36 1
						{
37 1
							new SiteAccessPathRule
38 1
							{
39 1
								Path = "/",
40 1
								RuleType = PathRuleType.Disallow
41 1
							}
42 1
						}
43 1
					}
44 1
				}
45 1
			};
46 1
		}
47

48
		public bool IsAllowedAccess(Uri uri, string userAgent)
49 1
		{
50 1
			if (!uri.IsAbsoluteUri)
51 1
			{
52 1
				uri = new Uri(BaseUri, uri);
53 1
			}
54

55 1
			var entry = GetEntryForUserAgent(userAgent);
56 1
			return PathComparisonUtility.IsAllowed(entry, uri);
57 1
		}
58

59
		public SiteAccessEntry GetEntryForUserAgent(string userAgent)
60 1
		{
61 1
			SiteAccessEntry globalEntry = null;
62

63 1
			foreach (var siteAccessEntry in SiteAccessEntries)
64 1
			{
65 1
				if (globalEntry == null && siteAccessEntry.UserAgents.Any(u => u == "*"))
66 1
				{
67 1
					globalEntry = siteAccessEntry;
68 1
				}
69

70 1
				if (siteAccessEntry.UserAgents.Any(u => userAgent.IndexOf(u, StringComparison.InvariantCultureIgnoreCase) != -1))
71 1
				{
72 1
					return siteAccessEntry;
73
				}
74 1
			}
75

76 1
			return globalEntry;
77 1
		}
78
	}
79
}

Read our documentation on viewing source code .

Loading